Skip to content

Commit

Permalink
fs: get_file: try linking from storage
Browse files Browse the repository at this point in the history
Required for iterative/dvc#9385
  • Loading branch information
efiop committed May 15, 2023
1 parent 2029b6f commit 43acd59
Showing 1 changed file with 26 additions and 5 deletions.
31 changes: 26 additions & 5 deletions src/dvc_data/fs.py
Original file line number Diff line number Diff line change
Expand Up @@ -54,21 +54,25 @@ def _get_fs_path(self, path: "AnyFSPath"):

for typ in ["cache", "remote", "data"]:
try:
data = self.index.storage_map.get_storage(entry, typ)
info = self.index.storage_map[entry.key]
storage = getattr(info, typ)
if not storage:
continue
data = storage.get(entry)
except (ValueError, StorageKeyError):
continue
if data:
fs, fs_path = data
if fs.exists(fs_path):
return fs, typ, fs_path
return typ, storage, fs, fs_path

raise FileNotFoundError

def open( # type: ignore
self, path: str, mode="r", encoding=None, **kwargs
): # pylint: disable=arguments-renamed, arguments-differ
cache_odb = kwargs.pop("cache_odb", None)
fs, typ, fspath = self._get_fs_path(path, **kwargs)
typ, _, fs, fspath = self._get_fs_path(path, **kwargs)

if cache_odb and typ == "remote":
from dvc_data.hashfile.build import _upload_file
Expand Down Expand Up @@ -126,13 +130,30 @@ def info(self, path, **kwargs):
def get_file( # pylint: disable=arguments-differ
self, rpath, lpath, callback=DEFAULT_CALLBACK, **kwargs
):
from dvc_objects.fs.generic import transfer
from dvc_objects.fs.local import LocalFileSystem

from dvc_data.index import ObjectStorage

try:
fs, _, path = self._get_fs_path(rpath)
_, storage, fs, path = self._get_fs_path(rpath)
except IsADirectoryError:
os.makedirs(lpath, exist_ok=True)
return None

fs.get_file(path, lpath, callback=callback, **kwargs)
if isinstance(storage, ObjectStorage) and isinstance(
fs, LocalFileSystem
):
transfer(
fs,
path,
fs,
os.fspath(lpath),
callback=callback,
links=storage.odb.cache_types,
)
else:
fs.get_file(path, lpath, callback=callback, **kwargs)

def checksum(self, path):
info = self.info(path)
Expand Down

0 comments on commit 43acd59

Please sign in to comment.