From 4c1f17213239a3c2cb70902468376274f551a836 Mon Sep 17 00:00:00 2001 From: Peter Rowlands Date: Tue, 20 Sep 2022 17:51:42 +0900 Subject: [PATCH 1/2] deps: bump dvc-data to 0.10.1 --- setup.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.cfg b/setup.cfg index 41f52e94990..079bb389435 100644 --- a/setup.cfg +++ b/setup.cfg @@ -66,7 +66,7 @@ install_requires = dvc-render==0.0.11 dvc-task==0.1.2 dvclive>=0.10.0 - dvc-data==0.10.0 + dvc-data==0.10.1 dvc-http==2.19.1 hydra-core>=1.1.0 iterative-telemetry==0.0.5 From e63cb31a667a028150f9c2b72dde923b5f4b0c41 Mon Sep 17 00:00:00 2001 From: skshetry Date: Tue, 20 Sep 2022 15:24:44 +0545 Subject: [PATCH 2/2] dvcfs: remove get-file implementation, make dvcfs non-cachable (#8331) dvcfs: remove get-file implementation --- dvc/fs/dvc.py | 35 ++------------------------- tests/func/test_fs.py | 55 ------------------------------------------- 2 files changed, 2 insertions(+), 88 deletions(-) diff --git a/dvc/fs/dvc.py b/dvc/fs/dvc.py index 840cd60cc77..2be4733bfe2 100644 --- a/dvc/fs/dvc.py +++ b/dvc/fs/dvc.py @@ -10,9 +10,7 @@ from fsspec.spec import AbstractFileSystem from funcy import cached_property, wrap_prop, wrap_with -from dvc.utils.fs import makedirs from dvc_objects.fs.base import FileSystem -from dvc_objects.fs.callbacks import DEFAULT_CALLBACK from dvc_objects.fs.path import Path from .data import DataFileSystem @@ -66,15 +64,7 @@ def _get_dvc_path(dvc_fs, subkey): class _DVCFileSystem(AbstractFileSystem): # pylint:disable=abstract-method - """DVC + git-tracked files fs. - - Args: - repo: DVC or git repo. - subrepos: traverse to subrepos (by default, it ignores subrepos) - repo_factory: A function to initialize subrepo with, default is Repo. - kwargs: Additional keyword arguments passed to the `DataFileSystem()`. - """ - + cachable = False root_marker = "/" def __init__( @@ -238,7 +228,7 @@ def _open( dvc_path = _get_dvc_path(dvc_fs, subkey) return dvc_fs.open(dvc_path, mode=mode) - def isdvc(self, path, **kwargs): + def isdvc(self, path, **kwargs) -> bool: key = self._get_key_from_relative(path) _, dvc_fs, subkey = self._get_subrepo_info(key) dvc_path = _get_dvc_path(dvc_fs, subkey) @@ -295,27 +285,6 @@ def ls( # pylint: disable=arguments-differ return infos - def get_file( # pylint: disable=arguments-differ - self, rpath, lpath, callback=DEFAULT_CALLBACK, **kwargs - ): - key = self._get_key_from_relative(rpath) - fs_path = self._from_key(key) - fs = self.repo.fs - - if self.isdir(rpath): - makedirs(lpath, exist_ok=True) - return None - - try: - fs.get_file(fs_path, lpath, callback=callback, **kwargs) - return - except FileNotFoundError: - _, dvc_fs, subkey = self._get_subrepo_info(key) - if not dvc_fs: - raise - dvc_path = _get_dvc_path(dvc_fs, subkey) - dvc_fs.get_file(dvc_path, lpath, callback=callback, **kwargs) - def info(self, path, **kwargs): key = self._get_key_from_relative(path) ignore_subrepos = kwargs.get("ignore_subrepos", True) diff --git a/tests/func/test_fs.py b/tests/func/test_fs.py index 13aba98fe88..d830f7b2f1a 100644 --- a/tests/func/test_fs.py +++ b/tests/func/test_fs.py @@ -1,7 +1,6 @@ import os from operator import itemgetter -from dvc.fs.callbacks import Callback from dvc.repo import Repo @@ -42,57 +41,3 @@ def test_walk_dont_ignore_subrepos(tmp_dir, scm, dvc): assert set(get_dirs(next(dvc_fs.walk(path)))) == {".dvc", "subdir", ".git"} assert set(get_dirs(next(scm_fs.walk("/")))) == {".dvc", "subdir"} - - -def test_callback_on_dvcfs(tmp_dir, dvc, scm, mocker): - tmp_dir.dvc_gen({"dir": {"bar": "bar"}}, commit="dvc") - tmp_dir.scm_gen({"dir": {"foo": "foo"}}, commit="git") - - fs = dvc.dvcfs - - callback = Callback() - fs.get( - "dir", - (tmp_dir / "dir2").fs_path, - callback=callback, - ) - - assert (tmp_dir / "dir2").read_text() == {"foo": "foo", "bar": "bar"} - assert callback.size == 2 - assert callback.value == 2 - - callback = Callback() - branch = mocker.spy(callback, "branch") - fs.get( - os.path.join("dir", "foo"), - (tmp_dir / "foo").fs_path, - callback=callback, - ) - - size = os.path.getsize(tmp_dir / "dir" / "foo") - assert (tmp_dir / "foo").read_text() == "foo" - assert callback.size == 1 - assert callback.value == 1 - - assert branch.call_count == 1 - assert branch.spy_return.size == size - assert branch.spy_return.value == size - - branch.reset_mock() - - callback = Callback() - branch = mocker.spy(callback, "branch") - fs.get( - os.path.join("dir", "bar"), - (tmp_dir / "bar").fs_path, - callback=callback, - ) - - size = os.path.getsize(tmp_dir / "dir" / "bar") - assert (tmp_dir / "bar").read_text() == "bar" - assert callback.size == 1 - assert callback.value == 1 - - assert branch.call_count == 1 - assert branch.spy_return.size == size - assert branch.spy_return.value == size