diff --git a/dvc/utils/fs.py b/dvc/utils/fs.py index f3bd706c61..8d9f400784 100644 --- a/dvc/utils/fs.py +++ b/dvc/utils/fs.py @@ -189,7 +189,8 @@ def copyfile(src, dest, no_progress_bar=False, name=None): dest = fspath_py35(dest) name = name if name else os.path.basename(dest) - total = os.stat(src).st_size + src_stat = os.stat(src) + total = src_stat.st_size if os.path.isdir(dest): dest = os.path.join(dest, os.path.basename(src)) @@ -211,6 +212,7 @@ def copyfile(src, dest, no_progress_bar=False, name=None): if not buf: break fdest_wrapped.write(buf) + os.chmod(dest, src_stat.st_mode) def walk_files(directory): diff --git a/tests/dir_helpers.py b/tests/dir_helpers.py index 1eac1e59c4..e69f72dfab 100644 --- a/tests/dir_helpers.py +++ b/tests/dir_helpers.py @@ -60,7 +60,6 @@ "tmp_dir", "scm", "dvc", - "local_remote", "run_copy", "erepo_dir", "git_dir", @@ -200,6 +199,31 @@ def branch(self, name, new=False): finally: self.scm.checkout(old) + @property + def local_remote_path(self): + return self.parent / (self.name + "_local_remote") + + @contextmanager + def local_remote_context(self): + self._require("dvc") + self.setup_remote() + with self.chdir(): + yield + self.dvc.push() + + def setup_remote(self, remote_url=None): + self._require("dvc") + + if not remote_url: + # local by default + remote_url = fspath(self.local_remote_path) + with self.dvc.config.edit() as conf: + conf["remote"]["upstream"] = {"url": remote_url} + conf["core"]["remote"] = "upstream" + if hasattr(self, "scm"): + self.scm_add([self.dvc.config.files["repo"]], commit="add remote") + return remote_url + def _coerce_filenames(filenames): if isinstance(filenames, (str, bytes, pathlib.PurePath)): @@ -257,17 +281,6 @@ def _git_init(path): git.close() -@pytest.fixture -def local_remote(request, tmp_dir, dvc, make_tmp_dir): - path = make_tmp_dir("local-remote") - with dvc.config.edit() as conf: - conf["remote"]["upstream"] = {"url": fspath(path)} - conf["core"]["remote"] = "upstream" - if "scm" in request.fixturenames: - tmp_dir.scm_add([dvc.config.files["repo"]], commit="add remote") - return path - - @pytest.fixture def run_copy(tmp_dir, dvc): tmp_dir.gen( @@ -288,17 +301,7 @@ def run_copy(src, dst, **run_kwargs): @pytest.fixture def erepo_dir(make_tmp_dir): - path = make_tmp_dir("erepo", scm=True, dvc=True) - - # Chdir for git and dvc to work locally - with path.chdir(): - with path.dvc.config.edit() as conf: - cache_dir = path.dvc.cache.local.cache_dir - conf["remote"]["upstream"] = {"url": cache_dir} - conf["core"]["remote"] = "upstream" - path.scm_add([path.dvc.config.files["repo"]], commit="add remote") - - return path + return make_tmp_dir("erepo", scm=True, dvc=True) @pytest.fixture diff --git a/tests/func/test_api.py b/tests/func/test_api.py index 943312fe07..bc26f68f07 100644 --- a/tests/func/test_api.py +++ b/tests/func/test_api.py @@ -39,7 +39,7 @@ def test_get_url(tmp_dir, dvc, remote_url): @pytest.mark.parametrize("remote_url", remote_params, indirect=True) def test_get_url_external(erepo_dir, remote_url): - _set_remote_url_and_commit(erepo_dir.dvc, remote_url) + erepo_dir.setup_remote(remote_url) with erepo_dir.chdir(): erepo_dir.dvc_gen("foo", "foo", commit="add foo") @@ -74,7 +74,7 @@ def test_open(remote_url, tmp_dir, dvc): @pytest.mark.parametrize("remote_url", all_remote_params, indirect=True) def test_open_external(remote_url, erepo_dir): - _set_remote_url_and_commit(erepo_dir.dvc, remote_url) + erepo_dir.setup_remote(remote_url) with erepo_dir.chdir(): erepo_dir.dvc_gen("version", "master", commit="add version") @@ -108,13 +108,6 @@ def test_missing(remote_url, tmp_dir, dvc): api.read("foo") -def _set_remote_url_and_commit(repo, remote_url): - with repo.config.edit() as conf: - conf["remote"]["upstream"]["url"] = remote_url - repo.scm.add([repo.config.files["repo"]]) - repo.scm.commit("modify remote") - - def test_open_scm_controlled(tmp_dir, erepo_dir): erepo_dir.scm_gen({"scm_controlled": "file content"}, commit="create file") diff --git a/tests/func/test_data_cloud.py b/tests/func/test_data_cloud.py index a234fb3ff3..d5222ee017 100644 --- a/tests/func/test_data_cloud.py +++ b/tests/func/test_data_cloud.py @@ -6,6 +6,7 @@ from unittest import SkipTest import pytest +from funcy import first from dvc.compat import fspath, fspath_py35 from dvc.cache import NamedCache @@ -688,14 +689,11 @@ def test(self): def test_verify_checksums(tmp_dir, scm, dvc, mocker, tmp_path_factory): - tmp_dir.dvc_gen({"file": "file1 content"}, commit="add file") - tmp_dir.dvc_gen({"dir": {"subfile": "file2 content"}}, commit="add dir") - - dvc.config["remote"]["local_remote"] = { - "url": fspath(tmp_path_factory.mktemp("local_remote")) - } - dvc.config["core"]["remote"] = "local_remote" - dvc.push() + with tmp_dir.local_remote_context(): + tmp_dir.dvc_gen({"file": "file1 content"}, commit="add file") + tmp_dir.dvc_gen( + {"dir": {"subfile": "file2 content"}}, commit="add dir" + ) # remove artifacts and cache to trigger fetching remove("file") @@ -710,7 +708,7 @@ def test_verify_checksums(tmp_dir, scm, dvc, mocker, tmp_path_factory): # Removing cache will invalidate existing state entries remove(dvc.cache.local.cache_dir) - dvc.config["remote"]["local_remote"]["verify"] = True + dvc.config["remote"]["upstream"]["verify"] = True dvc.pull() assert checksum_spy.call_count == 3 @@ -783,13 +781,13 @@ def recurse_list_dir(d): ] -def test_dvc_pull_pipeline_stages(tmp_dir, dvc, local_remote, run_copy): - (stage0,) = tmp_dir.dvc_gen("foo", "foo") - stage1 = run_copy("foo", "bar", single_stage=True) - stage2 = run_copy("bar", "foobar", name="copy-bar-foobar") - outs = ["foo", "bar", "foobar"] +def test_dvc_pull_pipeline_stages(tmp_dir, dvc, run_copy): + with tmp_dir.local_remote_context(): + (stage0,) = tmp_dir.dvc_gen("foo", "foo") + stage1 = run_copy("foo", "bar", single_stage=True) + stage2 = run_copy("bar", "foobar", name="copy-bar-foobar") + outs = ["foo", "bar", "foobar"] - dvc.push() clean(outs, dvc) dvc.pull() assert all((tmp_dir / file).exists() for file in outs) @@ -813,21 +811,21 @@ def test_dvc_pull_pipeline_stages(tmp_dir, dvc, local_remote, run_copy): assert set(stats["added"]) == set(outs) -def test_pipeline_file_target_ops(tmp_dir, dvc, local_remote, run_copy): - tmp_dir.dvc_gen("foo", "foo") - run_copy("foo", "bar", single_stage=True) +def test_pipeline_file_target_ops(tmp_dir, dvc, run_copy): + with tmp_dir.local_remote_context(): + tmp_dir.dvc_gen("foo", "foo") + run_copy("foo", "bar", single_stage=True) - tmp_dir.dvc_gen("lorem", "lorem") - run_copy("lorem", "lorem2", name="copy-lorem-lorem2") + tmp_dir.dvc_gen("lorem", "lorem") + run_copy("lorem", "lorem2", name="copy-lorem-lorem2") - tmp_dir.dvc_gen("ipsum", "ipsum") - run_copy("ipsum", "baz", name="copy-ipsum-baz") + tmp_dir.dvc_gen("ipsum", "ipsum") + run_copy("ipsum", "baz", name="copy-ipsum-baz") outs = ["foo", "bar", "lorem", "ipsum", "baz", "lorem2"] - dvc.push() # each one's a copy of other, hence 3 - assert len(recurse_list_dir(fspath_py35(local_remote))) == 3 + assert len(recurse_list_dir(fspath_py35(tmp_dir.local_remote_path))) == 3 clean(outs, dvc) assert set(dvc.pull(["dvc.yaml"])["added"]) == {"lorem2", "baz"} @@ -836,13 +834,13 @@ def test_pipeline_file_target_ops(tmp_dir, dvc, local_remote, run_copy): assert set(dvc.pull()["added"]) == set(outs) # clean everything in remote and push - clean(local_remote.iterdir()) + clean(tmp_dir.local_remote_path.iterdir()) dvc.push(["dvc.yaml:copy-ipsum-baz"]) - assert len(recurse_list_dir(fspath_py35(local_remote))) == 1 + assert len(recurse_list_dir(fspath_py35(tmp_dir.local_remote_path))) == 1 - clean(local_remote.iterdir()) + clean(tmp_dir.local_remote_path.iterdir()) dvc.push(["dvc.yaml"]) - assert len(recurse_list_dir(fspath_py35(local_remote))) == 2 + assert len(recurse_list_dir(fspath_py35(tmp_dir.local_remote_path))) == 2 with pytest.raises(StageNotFound): dvc.push(["dvc.yaml:StageThatDoesNotExist"]) @@ -859,8 +857,10 @@ def test_pipeline_file_target_ops(tmp_dir, dvc, local_remote, run_copy): ({}, "Everything is up to date"), ], ) -def test_push_stats(tmp_dir, dvc, fs, msg, local_remote, caplog): +def test_push_stats(tmp_dir, dvc, fs, msg, caplog): + tmp_dir.setup_remote() tmp_dir.dvc_gen(fs) + caplog.clear() with caplog.at_level(level=logging.INFO, logger="dvc"): main(["push"]) @@ -875,9 +875,9 @@ def test_push_stats(tmp_dir, dvc, fs, msg, local_remote, caplog): ({}, "Everything is up to date."), ], ) -def test_fetch_stats(tmp_dir, dvc, fs, msg, local_remote, caplog): - tmp_dir.dvc_gen(fs) - dvc.push() +def test_fetch_stats(tmp_dir, dvc, fs, msg, caplog): + with tmp_dir.local_remote_context(): + tmp_dir.dvc_gen(fs) clean(list(fs.keys()), dvc) caplog.clear() with caplog.at_level(level=logging.INFO, logger="dvc"): @@ -885,9 +885,9 @@ def test_fetch_stats(tmp_dir, dvc, fs, msg, local_remote, caplog): assert msg in caplog.text -def test_pull_stats(tmp_dir, dvc, local_remote, caplog): - tmp_dir.dvc_gen({"foo": "foo", "bar": "bar"}) - dvc.push() +def test_pull_stats(tmp_dir, dvc, caplog): + with tmp_dir.local_remote_context(): + tmp_dir.dvc_gen({"foo": "foo", "bar": "bar"}) clean(["foo", "bar"], dvc) (tmp_dir / "bar").write_text("foobar") caplog.clear() @@ -901,3 +901,16 @@ def test_pull_stats(tmp_dir, dvc, local_remote, caplog): with caplog.at_level(level=logging.INFO, logger="dvc"): main(["pull"]) assert "Everything is up to date." in caplog.text + + +def test_local_remote_should_retain_cache_mode(tmp_dir, dvc): + tmp_dir.setup_remote() + remote = dvc.cloud.get_remote("upstream") + + (stage,) = tmp_dir.dvc_gen("file", "file content") + out = first(stage.outs) + + dvc.push() + + remote_path = remote.checksum_to_path(out.checksum) + assert os.stat(remote_path).st_mode == os.stat(out.cache_path).st_mode diff --git a/tests/func/test_external_repo.py b/tests/func/test_external_repo.py index 2455a3352e..02e1b61108 100644 --- a/tests/func/test_external_repo.py +++ b/tests/func/test_external_repo.py @@ -44,7 +44,7 @@ def test_source_change(erepo_dir): def test_cache_reused(erepo_dir, mocker): - with erepo_dir.chdir(): + with erepo_dir.local_remote_context(): erepo_dir.dvc_gen("file", "text", commit="add file") download_spy = mocker.spy(LocalRemote, "download") @@ -63,6 +63,8 @@ def test_cache_reused(erepo_dir, mocker): def test_known_sha(erepo_dir): + erepo_dir.scm.commit("init") + url = "file://{}".format(erepo_dir) with external_repo(url) as repo: rev = repo.scm.get_rev() @@ -100,9 +102,7 @@ def test_relative_remote(erepo_dir, tmp_dir): upstream_dir = tmp_dir upstream_url = relpath(upstream_dir, erepo_dir) - with erepo_dir.dvc.config.edit() as conf: - conf["remote"]["upstream"] = {"url": upstream_url} - conf["core"]["remote"] = "upstream" + erepo_dir.setup_remote(upstream_url) erepo_dir.scm_add( erepo_dir.dvc.config.files["repo"], commit="Update dvc config" diff --git a/tests/func/test_gc.py b/tests/func/test_gc.py index 0698bea794..5a1e7ebc9b 100644 --- a/tests/func/test_gc.py +++ b/tests/func/test_gc.py @@ -240,7 +240,7 @@ def test_gc_without_workspace_raises_error(tmp_dir, dvc): def test_gc_cloud_with_or_without_specifier(tmp_dir, erepo_dir): dvc = erepo_dir.dvc - with erepo_dir.chdir(): + with erepo_dir.local_remote_context(): from dvc.exceptions import InvalidArgumentError with pytest.raises(InvalidArgumentError): @@ -296,12 +296,7 @@ def test_gc_with_possible_args_positive(tmp_dir, dvc): def test_gc_cloud_positive(tmp_dir, dvc, tmp_path_factory): - with dvc.config.edit() as conf: - storage = fspath(tmp_path_factory.mktemp("test_remote_base")) - conf["remote"]["local_remote"] = {"url": storage} - conf["core"]["remote"] = "local_remote" - - dvc.push() + tmp_dir.setup_remote() for flag in ["-cw", "-ca", "-cT", "-caT", "-cwT"]: assert main(["gc", "-vf", flag]) == 0 diff --git a/tests/func/test_get.py b/tests/func/test_get.py index 582212ed2c..08b1e4096c 100644 --- a/tests/func/test_get.py +++ b/tests/func/test_get.py @@ -198,7 +198,7 @@ def test_get_file_from_dir(tmp_dir, erepo_dir): def test_get_url_positive(tmp_dir, erepo_dir, caplog): - with erepo_dir.chdir(): + with erepo_dir.local_remote_context(): erepo_dir.dvc_gen("foo", "foo") caplog.clear() @@ -224,16 +224,14 @@ def test_get_url_git_only_repo(tmp_dir, scm, caplog): assert "failed to show URL" in caplog.text -def test_get_pipeline_tracked_outs( - tmp_dir, dvc, scm, git_dir, local_remote, run_copy -): +def test_get_pipeline_tracked_outs(tmp_dir, dvc, scm, git_dir, run_copy): from dvc.dvcfile import PIPELINE_FILE, PIPELINE_LOCK - tmp_dir.gen("foo", "foo") - run_copy("foo", "bar", name="copy-foo-bar") + with tmp_dir.local_remote_context(): + tmp_dir.gen("foo", "foo") + run_copy("foo", "bar", name="copy-foo-bar") dvc.scm.add([PIPELINE_FILE, PIPELINE_LOCK]) dvc.scm.commit("add pipeline stage") - dvc.push() with git_dir.chdir(): Repo.get("file:///{}".format(fspath(tmp_dir)), "bar", out="baz") diff --git a/tests/func/test_import.py b/tests/func/test_import.py index 53d28b88b5..68cf97a0c2 100644 --- a/tests/func/test_import.py +++ b/tests/func/test_import.py @@ -314,28 +314,25 @@ def test_import_from_bare_git_repo(tmp_dir, make_tmp_dir, erepo_dir): git.Repo.init(fspath(tmp_dir), bare=True) - with erepo_dir.chdir(): + with erepo_dir.local_remote_context(): erepo_dir.dvc_gen({"foo": "foo"}, commit="initial") - erepo_dir.dvc.push() - erepo_dir.scm.repo.create_remote("origin", fspath(tmp_dir)) - erepo_dir.scm.repo.remote("origin").push("master") + erepo_dir.scm.repo.create_remote("origin", fspath(tmp_dir)) + erepo_dir.scm.repo.remote("origin").push("master") dvc_repo = make_tmp_dir("dvc-repo", scm=True, dvc=True) with dvc_repo.chdir(): dvc_repo.dvc.imp(fspath(tmp_dir), "foo") -def test_import_pipeline_tracked_outs( - tmp_dir, dvc, scm, erepo_dir, local_remote, run_copy -): +def test_import_pipeline_tracked_outs(tmp_dir, dvc, scm, erepo_dir, run_copy): from dvc.dvcfile import PIPELINE_FILE, PIPELINE_LOCK - tmp_dir.gen("foo", "foo") - run_copy("foo", "bar", name="copy-foo-bar") + with tmp_dir.local_remote_context(): + tmp_dir.gen("foo", "foo") + run_copy("foo", "bar", name="copy-foo-bar") dvc.scm.add([PIPELINE_FILE, PIPELINE_LOCK]) dvc.scm.commit("add pipeline stage") - dvc.push() with erepo_dir.chdir(): erepo_dir.dvc.imp( diff --git a/tests/func/test_remote.py b/tests/func/test_remote.py index 946ad69f10..814d635487 100644 --- a/tests/func/test_remote.py +++ b/tests/func/test_remote.py @@ -173,9 +173,7 @@ def test_dir_checksum_should_be_key_order_agnostic(tmp_dir, dvc): def test_partial_push_n_pull(tmp_dir, dvc, tmp_path_factory): - url = fspath(tmp_path_factory.mktemp("upstream")) - dvc.config["remote"]["upstream"] = {"url": url} - dvc.config["core"]["remote"] = "upstream" + tmp_dir.setup_remote() foo = tmp_dir.dvc_gen({"foo": "foo content"})[0].outs[0] bar = tmp_dir.dvc_gen({"bar": "bar content"})[0].outs[0] @@ -212,10 +210,7 @@ def unreliable_upload(self, from_file, to_info, name=None, **kwargs): def test_raise_on_too_many_open_files(tmp_dir, dvc, tmp_path_factory, mocker): - storage = fspath(tmp_path_factory.mktemp("test_remote_base")) - dvc.config["remote"]["local_remote"] = {"url": storage} - dvc.config["core"]["remote"] = "local_remote" - + tmp_dir.setup_remote() tmp_dir.dvc_gen({"file": "file content"}) mocker.patch.object( @@ -245,10 +240,7 @@ def test_external_dir_resource_on_no_cache(tmp_dir, dvc, tmp_path_factory): def test_push_order(tmp_dir, dvc, tmp_path_factory, mocker): - url = fspath(tmp_path_factory.mktemp("upstream")) - dvc.config["remote"]["upstream"] = {"url": url} - dvc.config["core"]["remote"] = "upstream" - + tmp_dir.setup_remote() tmp_dir.dvc_gen({"foo": {"bar": "bar content"}}) tmp_dir.dvc_gen({"baz": "baz content"})