From c45b3992767087242fffa8cf8ca6ff5d72a7ae68 Mon Sep 17 00:00:00 2001 From: Alexander Schepanovski Date: Sun, 15 Dec 2019 13:51:25 +0700 Subject: [PATCH 1/2] dvc: use dynamic scope to stop callback/flag passing --- dvc/output/base.py | 15 ++---- dvc/progress.py | 40 ++++++++++++++++ dvc/remote/azure.py | 12 ++--- dvc/remote/base.py | 91 ++++++++++-------------------------- dvc/remote/gdrive.py | 29 ++++-------- dvc/remote/gs.py | 28 ++--------- dvc/remote/http.py | 6 +-- dvc/remote/local.py | 16 ++----- dvc/remote/oss.py | 12 ++--- dvc/remote/s3.py | 16 +++---- dvc/remote/ssh/__init__.py | 20 +++----- dvc/remote/ssh/connection.py | 16 ++----- dvc/repo/checkout.py | 27 +++++------ dvc/stage.py | 9 +--- dvc/utils/__init__.py | 6 +-- 15 files changed, 129 insertions(+), 214 deletions(-) diff --git a/dvc/output/base.py b/dvc/output/base.py index ffb68ac6af..dd9a481803 100644 --- a/dvc/output/base.py +++ b/dvc/output/base.py @@ -6,6 +6,7 @@ from voluptuous import Any import dvc.prompt as prompt +from dvc.progress import flags, noop from dvc.cache import NamedCache from dvc.exceptions import CollectCacheError from dvc.exceptions import DvcException @@ -282,12 +283,10 @@ def verify_metric(self): def download(self, to): self.remote.download(self.path_info, to.path_info) - def checkout( - self, force=False, progress_callback=None, tag=None, relink=False - ): + def checkout(self, force=False, tag=None, relink=False): if not self.use_cache: - if progress_callback: - progress_callback(str(self.path_info), self.get_files_number()) + pbar = flags.tqdm or noop + pbar.update_desc(str(self), self.get_files_number()) return None if tag: @@ -296,11 +295,7 @@ def checkout( info = self.info return self.cache.checkout( - self.path_info, - info, - force=force, - progress_callback=progress_callback, - relink=relink, + self.path_info, info, force=force, relink=relink ) def remove(self, ignore_remove=False): diff --git a/dvc/progress.py b/dvc/progress.py index b11fb34f3f..60ed3e4e7c 100644 --- a/dvc/progress.py +++ b/dvc/progress.py @@ -68,6 +68,9 @@ def __init__( if file is None: file = sys.stderr self.desc_persist = desc + # disable by context flag + if flags.tqdm_disable: + disable = True # auto-disable based on `logger.level` if disable is None: disable = logger.getEffectiveLevel() > level @@ -131,3 +134,40 @@ def format_dict(self): d["ncols_desc"] = 1 d["prefix"] = "" return d + + +# Adding this here for showcase only +from contextlib import contextmanager # noqa + + +class ContextFlags: + def __init__(self): + self._stack = [] + + def __call__(self, **values): + return _flags(self._stack, values) + + def __getattr__(self, name): + for d in reversed(self._stack): + if name in d: + return d[name] + else: + return None + + +flags = ContextFlags() + + +@contextmanager +def _flags(stack, values): + stack.append(values) + yield + stack.pop() + + +class Noop: + def __getattr__(self, name): + return lambda self, *a, **kw: None + + +noop = Noop() diff --git a/dvc/remote/azure.py b/dvc/remote/azure.py index b991195dfe..da8a5e0c95 100644 --- a/dvc/remote/azure.py +++ b/dvc/remote/azure.py @@ -110,10 +110,8 @@ def _list_paths(self, bucket, prefix): def list_cache_paths(self): return self._list_paths(self.path_info.bucket, self.path_info.path) - def _upload( - self, from_file, to_info, name=None, no_progress_bar=False, **_kwargs - ): - with Tqdm(desc=name, disable=no_progress_bar, bytes=True) as pbar: + def _upload(self, from_file, to_info, name=None, **_kwargs): + with Tqdm(desc=name, bytes=True) as pbar: self.blob_service.create_blob_from_path( to_info.bucket, to_info.path, @@ -121,10 +119,8 @@ def _upload( progress_callback=pbar.update_to, ) - def _download( - self, from_info, to_file, name=None, no_progress_bar=False, **_kwargs - ): - with Tqdm(desc=name, disable=no_progress_bar, bytes=True) as pbar: + def _download(self, from_info, to_file, name=None, **_kwargs): + with Tqdm(desc=name, bytes=True) as pbar: self.blob_service.get_blob_to_path( from_info.bucket, from_info.path, diff --git a/dvc/remote/base.py b/dvc/remote/base.py index f72e56127d..a7321d7a90 100644 --- a/dvc/remote/base.py +++ b/dvc/remote/base.py @@ -22,7 +22,7 @@ ) from dvc.ignore import DvcIgnore from dvc.path_info import PathInfo, URLInfo -from dvc.progress import Tqdm +from dvc.progress import Tqdm, flags, noop from dvc.remote.slow_link_detection import slow_link_guard from dvc.state import StateNoop from dvc.utils import makedirs, relpath, tmp_fname @@ -238,7 +238,8 @@ def _get_dir_info_checksum(self, dir_info): from_info = PathInfo(tmp) to_info = self.cache.path_info / tmp_fname("") - self.cache.upload(from_info, to_info, no_progress_bar=True) + with flags(tqdm_disable=True): + self.cache.upload(from_info, to_info) checksum = self.get_file_checksum(to_info) + self.CHECKSUM_DIR_SUFFIX return checksum, to_info @@ -516,7 +517,7 @@ def _save(self, path_info, checksum): return self._save_file(path_info, checksum) - def upload(self, from_info, to_info, name=None, no_progress_bar=False): + def upload(self, from_info, to_info, name=None): if not hasattr(self, "_upload"): raise RemoteActionNotImplemented("upload", self.scheme) @@ -531,12 +532,7 @@ def upload(self, from_info, to_info, name=None, no_progress_bar=False): name = name or from_info.name try: - self._upload( - from_info.fspath, - to_info, - name=name, - no_progress_bar=no_progress_bar, - ) + self._upload(from_info.fspath, to_info, name=name) except Exception: msg = "failed to upload '{}' to '{}'" logger.exception(msg.format(from_info, to_info)) @@ -545,13 +541,7 @@ def upload(self, from_info, to_info, name=None, no_progress_bar=False): return 0 def download( - self, - from_info, - to_info, - name=None, - no_progress_bar=False, - file_mode=None, - dir_mode=None, + self, from_info, to_info, name=None, file_mode=None, dir_mode=None ): if not hasattr(self, "_download"): raise RemoteActionNotImplemented("download", self.scheme) @@ -568,15 +558,13 @@ def download( if self.isdir(from_info): return self._download_dir( - from_info, to_info, name, no_progress_bar, file_mode, dir_mode + from_info, to_info, name, file_mode, dir_mode ) return self._download_file( - from_info, to_info, name, no_progress_bar, file_mode, dir_mode + from_info, to_info, name, file_mode, dir_mode ) - def _download_dir( - self, from_info, to_info, name, no_progress_bar, file_mode, dir_mode - ): + def _download_dir(self, from_info, to_info, name, file_mode, dir_mode): from_infos = list(self.walk_files(from_info)) to_infos = ( to_info / info.relative_to(from_info) for info in from_infos @@ -586,7 +574,6 @@ def _download_dir( download_files = partial( self._download_file, name=name, - no_progress_bar=True, file_mode=file_mode, dir_mode=dir_mode, ) @@ -596,13 +583,10 @@ def _download_dir( total=len(from_infos), desc="Downloading directory", unit="Files", - disable=no_progress_bar, - ) as futures: + ) as futures, flags(tqdm_disable=True): return sum(futures) - def _download_file( - self, from_info, to_info, name, no_progress_bar, file_mode, dir_mode - ): + def _download_file(self, from_info, to_info, name, file_mode, dir_mode): makedirs(to_info.parent, exist_ok=True, mode=dir_mode) logger.debug("Downloading '{}' to '{}'".format(from_info, to_info)) @@ -611,9 +595,7 @@ def _download_file( tmp_file = tmp_fname(to_info) try: - self._download( - from_info, tmp_file, name=name, no_progress_bar=no_progress_bar - ) + self._download(from_info, tmp_file, name=name) except Exception: msg = "failed to download '{}' to '{}'" logger.exception(msg.format(from_info, to_info)) @@ -813,9 +795,7 @@ def safe_remove(self, path_info, force=False): self.remove(path_info) - def _checkout_file( - self, path_info, checksum, force, progress_callback=None - ): + def _checkout_file(self, path_info, checksum, force): """The file is changed we need to checkout a new copy""" cache_info = self.checksum_to_path_info(checksum) if self.exists(path_info): @@ -826,8 +806,7 @@ def _checkout_file( self.link(cache_info, path_info) self.state.save_link(path_info) self.state.save(path_info, checksum) - if progress_callback: - progress_callback(str(path_info)) + (flags.tqdm or noop).update_desc(str(path_info)) def makedirs(self, path_info): """Optional: Implement only if the remote needs to create @@ -835,9 +814,7 @@ def makedirs(self, path_info): """ pass - def _checkout_dir( - self, path_info, checksum, force, progress_callback=None, relink=False - ): + def _checkout_dir(self, path_info, checksum, force, relink=False): # Create dir separately so that dir is created # even if there are no files in it if not self.exists(path_info): @@ -858,8 +835,8 @@ def _checkout_dir( self.safe_remove(entry_info, force=force) self.link(entry_cache_info, entry_info) self.state.save(entry_info, entry_checksum) - if progress_callback: - progress_callback(str(entry_info)) + + (flags.tqdm or noop).update_desc(str(entry_info)) self._remove_redundant_files(path_info, dir_info, force) @@ -876,14 +853,7 @@ def _remove_redundant_files(self, path_info, dir_info, force): for path in existing_files - needed_files: self.safe_remove(path, force) - def checkout( - self, - path_info, - checksum_info, - force=False, - progress_callback=None, - relink=False, - ): + def checkout(self, path_info, checksum_info, force=False, relink=False): if path_info.scheme not in ["local", self.scheme]: raise NotImplementedError @@ -910,33 +880,20 @@ def checkout( failed = path_info if failed or skip: - if progress_callback: - progress_callback( - str(path_info), self.get_files_number(checksum) - ) + pbar = flags.tqdm or noop + pbar.update_desc(str(path_info), self.get_files_number(checksum)) return failed msg = "Checking out '{}' with cache '{}'." logger.debug(msg.format(str(path_info), checksum)) - self._checkout(path_info, checksum, force, progress_callback, relink) + self._checkout(path_info, checksum, force, relink) return None - def _checkout( - self, - path_info, - checksum, - force=False, - progress_callback=None, - relink=False, - ): + def _checkout(self, path_info, checksum, force=False, relink=False): if not self.is_dir_checksum(checksum): - return self._checkout_file( - path_info, checksum, force, progress_callback=progress_callback - ) - return self._checkout_dir( - path_info, checksum, force, progress_callback, relink - ) + return self._checkout_file(path_info, checksum, force) + return self._checkout_dir(path_info, checksum, force, relink) def get_files_number(self, checksum): if not checksum: diff --git a/dvc/remote/gdrive.py b/dvc/remote/gdrive.py index 75251c9d2b..10f6973685 100644 --- a/dvc/remote/gdrive.py +++ b/dvc/remote/gdrive.py @@ -89,9 +89,7 @@ def init_drive(self): ) @gdrive_retry - def gdrive_upload_file( - self, args, no_progress_bar=True, from_file="", progress_name="" - ): + def gdrive_upload_file(self, args, from_file, name): item = self.drive.CreateFile( {"title": args["title"], "parents": [{"id": args["parent_id"]}]} ) @@ -99,11 +97,7 @@ def gdrive_upload_file( with open(from_file, "rb") as fobj: total = os.path.getsize(from_file) with Tqdm.wrapattr( - fobj, - "read", - desc=progress_name, - total=total, - disable=no_progress_bar, + fobj, "read", desc=name, total=total ) as wrapped: # PyDrive doesn't like content property setting for empty files # https://github.com/gsuitedevs/PyDrive/issues/121 @@ -113,17 +107,13 @@ def gdrive_upload_file( return item @gdrive_retry - def gdrive_download_file( - self, file_id, to_file, progress_name, no_progress_bar - ): + def gdrive_download_file(self, file_id, to_file, name): gdrive_file = self.drive.CreateFile({"id": file_id}) bar_format = ( "Donwloading {desc:{ncols_desc}.{ncols_desc}}... " + Tqdm.format_sizeof(int(gdrive_file["fileSize"]), "B", 1024) ) - with Tqdm( - bar_format=bar_format, desc=progress_name, disable=no_progress_bar - ): + with Tqdm(bar_format=bar_format, desc=name): gdrive_file.GetContentFile(to_file) def gdrive_list_item(self, query): @@ -271,7 +261,7 @@ def get_remote_id(self, path_info, create=False): def exists(self, path_info): return self.get_remote_id(path_info) != "" - def _upload(self, from_file, to_info, name, no_progress_bar): + def _upload(self, from_file, to_info, name=None): dirname = to_info.parent if dirname: parent_id = self.get_remote_id(dirname, True) @@ -279,15 +269,12 @@ def _upload(self, from_file, to_info, name, no_progress_bar): parent_id = to_info.bucket self.gdrive_upload_file( - {"title": to_info.name, "parent_id": parent_id}, - no_progress_bar, - from_file, - name, + {"title": to_info.name, "parent_id": parent_id}, from_file, name ) - def _download(self, from_info, to_file, name, no_progress_bar): + def _download(self, from_info, to_file, name=None): file_id = self.get_remote_id(from_info) - self.gdrive_download_file(file_id, to_file, name, no_progress_bar) + self.gdrive_download_file(file_id, to_file, name) def all(self): if not self.cached_ids: diff --git a/dvc/remote/gs.py b/dvc/remote/gs.py index 3447da35e0..f2e04a1fbf 100644 --- a/dvc/remote/gs.py +++ b/dvc/remote/gs.py @@ -49,14 +49,7 @@ def wrapper(*args, **kwargs): @dynamic_chunk_size -def _upload_to_bucket( - bucket, - from_file, - to_info, - chunk_size=None, - name=None, - no_progress_bar=True, -): +def _upload_to_bucket(bucket, from_file, to_info, name=None, chunk_size=None): blob = bucket.blob(to_info.path, chunk_size=chunk_size) with io.open(from_file, mode="rb") as fobj: with Tqdm.wrapattr( @@ -64,7 +57,6 @@ def _upload_to_bucket( "read", desc=name or to_info.path, total=os.path.getsize(from_file), - disable=no_progress_bar, ) as wrapped: blob.upload_from_file(wrapped) @@ -170,26 +162,16 @@ def exists(self, path_info): """ return self.isfile(path_info) or self.isdir(path_info) - def _upload(self, from_file, to_info, name=None, no_progress_bar=True): + def _upload(self, from_file, to_info, name=None): bucket = self.gs.bucket(to_info.bucket) - _upload_to_bucket( - bucket, - from_file, - to_info, - name=name, - no_progress_bar=no_progress_bar, - ) + _upload_to_bucket(bucket, from_file, to_info, name=name) - def _download(self, from_info, to_file, name=None, no_progress_bar=True): + def _download(self, from_info, to_file, name=None): bucket = self.gs.bucket(from_info.bucket) blob = bucket.get_blob(from_info.path) with io.open(to_file, mode="wb") as fobj: with Tqdm.wrapattr( - fobj, - "write", - desc=name or from_info.path, - total=blob.size, - disable=no_progress_bar, + fobj, "write", desc=name or from_info.path, total=blob.size ) as wrapped: blob.download_to_file(wrapped) diff --git a/dvc/remote/http.py b/dvc/remote/http.py index 98a1e50f55..b1bf4d6e05 100644 --- a/dvc/remote/http.py +++ b/dvc/remote/http.py @@ -36,16 +36,16 @@ def __init__(self, repo, config): "files. Use: `dvc remote modify no_traverse true`" ) - def _download(self, from_info, to_file, name=None, no_progress_bar=False): + def _download(self, from_info, to_file, name=None): response = self._request("GET", from_info.url, stream=True) if response.status_code != 200: raise HTTPError(response.status_code, response.reason) + with Tqdm( - total=None if no_progress_bar else self._content_length(response), + total=self._content_length(response), leave=False, bytes=True, desc=from_info.url if name is None else name, - disable=no_progress_bar, ) as pbar: with open(to_file, "wb") as fd: for chunk in response.iter_content(chunk_size=self.CHUNK_SIZE): diff --git a/dvc/remote/local.py b/dvc/remote/local.py index 9c68e3a808..e452ed7e2f 100644 --- a/dvc/remote/local.py +++ b/dvc/remote/local.py @@ -219,23 +219,15 @@ def cache_exists(self, checksums, jobs=None, name=None): if not self.changed_cache_file(checksum) ] - def _upload( - self, from_file, to_info, name=None, no_progress_bar=False, **_kwargs - ): + def _upload(self, from_file, to_info, name=None, **_kwargs): makedirs(to_info.parent, exist_ok=True) tmp_file = tmp_fname(to_info) - copyfile( - from_file, tmp_file, name=name, no_progress_bar=no_progress_bar - ) + copyfile(from_file, tmp_file, name=name) os.rename(tmp_file, fspath_py35(to_info)) - def _download( - self, from_info, to_file, name=None, no_progress_bar=False, **_kwargs - ): - copyfile( - from_info, to_file, no_progress_bar=no_progress_bar, name=name - ) + def _download(self, from_info, to_file, name=None, **_kwargs): + copyfile(from_info, to_file, name=name) @staticmethod def open(path_info, mode="r", encoding=None): diff --git a/dvc/remote/oss.py b/dvc/remote/oss.py index 9445fd0b2b..5ff3aee558 100644 --- a/dvc/remote/oss.py +++ b/dvc/remote/oss.py @@ -104,18 +104,14 @@ def _list_paths(self, prefix): def list_cache_paths(self): return self._list_paths(self.path_info.path) - def _upload( - self, from_file, to_info, name=None, no_progress_bar=False, **_kwargs - ): - with Tqdm(desc=name, disable=no_progress_bar, bytes=True) as pbar: + def _upload(self, from_file, to_info, name=None, **_kwargs): + with Tqdm(desc=name, bytes=True) as pbar: self.oss_service.put_object_from_file( to_info.path, from_file, progress_callback=pbar.update_to ) - def _download( - self, from_info, to_file, name=None, no_progress_bar=False, **_kwargs - ): - with Tqdm(desc=name, disable=no_progress_bar, bytes=True) as pbar: + def _download(self, from_info, to_file, name=None, **_kwargs): + with Tqdm(desc=name, bytes=True) as pbar: self.oss_service.get_object_to_file( from_info.path, to_file, progress_callback=pbar.update_to ) diff --git a/dvc/remote/s3.py b/dvc/remote/s3.py index 6366fc426c..dfc742df8e 100644 --- a/dvc/remote/s3.py +++ b/dvc/remote/s3.py @@ -11,7 +11,7 @@ from dvc.exceptions import DvcException from dvc.exceptions import ETagMismatchError from dvc.path_info import CloudURLInfo -from dvc.progress import Tqdm +from dvc.progress import Tqdm, flags from dvc.remote.base import RemoteBASE from dvc.scheme import Schemes @@ -265,11 +265,9 @@ def isdir(self, path_info): dir_path = path_info / "" return bool(list(self._list_paths(dir_path, max_items=1))) - def _upload(self, from_file, to_info, name=None, no_progress_bar=False): + def _upload(self, from_file, to_info, name=None): total = os.path.getsize(from_file) - with Tqdm( - disable=no_progress_bar, total=total, bytes=True, desc=name - ) as pbar: + with Tqdm(total=total, bytes=True, desc=name) as pbar: self.s3.upload_file( from_file, to_info.bucket, @@ -278,16 +276,14 @@ def _upload(self, from_file, to_info, name=None, no_progress_bar=False): ExtraArgs=self.extra_args, ) - def _download(self, from_info, to_file, name=None, no_progress_bar=False): - if no_progress_bar: + def _download(self, from_info, to_file, name=None): + if flags.tqdm_disable: total = None else: total = self.s3.head_object( Bucket=from_info.bucket, Key=from_info.path )["ContentLength"] - with Tqdm( - disable=no_progress_bar, total=total, bytes=True, desc=name - ) as pbar: + with Tqdm(total=total, bytes=True, desc=name) as pbar: self.s3.download_file( from_info.bucket, from_info.path, to_file, Callback=pbar.update ) diff --git a/dvc/remote/ssh/__init__.py b/dvc/remote/ssh/__init__.py index 78da815e1a..7c5d3ec696 100644 --- a/dvc/remote/ssh/__init__.py +++ b/dvc/remote/ssh/__init__.py @@ -228,25 +228,17 @@ def move(self, from_info, to_info): with self.ssh(from_info) as ssh: ssh.move(from_info.path, to_info.path) - def _download(self, from_info, to_file, name=None, no_progress_bar=False): + def _download(self, from_info, to_file, name=None): assert from_info.isin(self.path_info) + with self.ssh(self.path_info) as ssh: - ssh.download( - from_info.path, - to_file, - progress_title=name, - no_progress_bar=no_progress_bar, - ) + ssh.download(from_info.path, to_file, name=name) - def _upload(self, from_file, to_info, name=None, no_progress_bar=False): + def _upload(self, from_file, to_info, name=None): assert to_info.isin(self.path_info) + with self.ssh(self.path_info) as ssh: - ssh.upload( - from_file, - to_info.path, - progress_title=name, - no_progress_bar=no_progress_bar, - ) + ssh.upload(from_file, to_info.path, name=name) @contextmanager def open(self, path_info, mode="r", encoding=None): diff --git a/dvc/remote/ssh/connection.py b/dvc/remote/ssh/connection.py index abc131173b..fa7d65b8af 100644 --- a/dvc/remote/ssh/connection.py +++ b/dvc/remote/ssh/connection.py @@ -177,12 +177,8 @@ def remove(self, path): else: self._remove_file(path) - def download(self, src, dest, no_progress_bar=False, progress_title=None): - with Tqdm( - desc=progress_title or os.path.basename(src), - disable=no_progress_bar, - bytes=True, - ) as pbar: + def download(self, src, dest, name=None): + with Tqdm(desc=name or os.path.basename(src), bytes=True) as pbar: self.sftp.get(src, dest, callback=pbar.update_to) def move(self, src, dst): @@ -207,15 +203,11 @@ def atomic_copy(self, src, dst): finally: self.remove(tmp) - def upload(self, src, dest, no_progress_bar=False, progress_title=None): + def upload(self, src, dest, name=None): self.makedirs(posixpath.dirname(dest)) tmp_file = tmp_fname(dest) - if not progress_title: - progress_title = posixpath.basename(dest) - with Tqdm( - desc=progress_title, disable=no_progress_bar, bytes=True - ) as pbar: + with Tqdm(desc=name or posixpath.basename(dest), bytes=True) as pbar: self.sftp.put(src, tmp_file, callback=pbar.update_to) self.sftp.rename(tmp_file, dest) diff --git a/dvc/repo/checkout.py b/dvc/repo/checkout.py index 040590d86c..3a6bb3c828 100644 --- a/dvc/repo/checkout.py +++ b/dvc/repo/checkout.py @@ -2,9 +2,11 @@ import logging +from funcy.py3 import lcat + from dvc.exceptions import CheckoutError from dvc.exceptions import CheckoutErrorSuggestGit -from dvc.progress import Tqdm +from dvc.progress import Tqdm, flags logger = logging.getLogger(__name__) @@ -54,17 +56,12 @@ def _checkout( total = get_all_files_numbers(stages) if total == 0: logger.info("Nothing to do") - failed = [] - with Tqdm( - total=total, unit="file", desc="Checkout", disable=total == 0 - ) as pbar: - for stage in stages: - failed.extend( - stage.checkout( - force=force, - progress_callback=pbar.update_desc, - relink=relink, - ) - ) - if failed: - raise CheckoutError(failed) + + with Tqdm(total=total, unit="file", desc="Checkout") as pbar, flags( + tqdm=pbar + ): + failed = lcat( + stage.checkout(force=force, relink=relink) for stage in stages + ) + if failed: + raise CheckoutError(failed) diff --git a/dvc/stage.py b/dvc/stage.py index 236a5f2d39..c35146231e 100644 --- a/dvc/stage.py +++ b/dvc/stage.py @@ -960,15 +960,10 @@ def check_missing_outputs(self): raise MissingDataSource(paths) @rwlocked(write=["outs"]) - def checkout(self, force=False, progress_callback=None, relink=False): + def checkout(self, force=False, relink=False): failed_checkouts = [] for out in self.outs: - failed = out.checkout( - force=force, - tag=self.tag, - progress_callback=progress_callback, - relink=relink, - ) + failed = out.checkout(force=force, tag=self.tag, relink=relink) if failed: failed_checkouts.append(failed) return failed_checkouts diff --git a/dvc/utils/__init__.py b/dvc/utils/__init__.py index a1d15aa0b3..df3d058ba6 100644 --- a/dvc/utils/__init__.py +++ b/dvc/utils/__init__.py @@ -112,7 +112,7 @@ def dict_md5(d, exclude=()): return bytes_md5(byts) -def copyfile(src, dest, no_progress_bar=False, name=None): +def copyfile(src, dest, name=None): """Copy file with progress bar""" from dvc.exceptions import DvcException from dvc.progress import Tqdm @@ -130,9 +130,7 @@ def copyfile(src, dest, no_progress_bar=False, name=None): try: System.reflink(src, dest) except DvcException: - with Tqdm( - desc=name, disable=no_progress_bar, total=total, bytes=True - ) as pbar: + with Tqdm(desc=name, total=total, bytes=True) as pbar: with open(src, "rb") as fsrc, open(dest, "wb+") as fdest: while True: buf = fsrc.read(LOCAL_CHUNK_SIZE) From a9983f9c9f2778e14f6094b436770dd32f57e515 Mon Sep 17 00:00:00 2001 From: Alexander Schepanovski Date: Sun, 15 Dec 2019 20:40:57 +0700 Subject: [PATCH 2/2] checkout: explain weird code in Repo.checkout() --- dvc/repo/checkout.py | 1 + 1 file changed, 1 insertion(+) diff --git a/dvc/repo/checkout.py b/dvc/repo/checkout.py index 3a6bb3c828..6117e8c3d6 100644 --- a/dvc/repo/checkout.py +++ b/dvc/repo/checkout.py @@ -56,6 +56,7 @@ def _checkout( total = get_all_files_numbers(stages) if total == 0: logger.info("Nothing to do") + # We might need to create empty dirs though, so no return here with Tqdm(total=total, unit="file", desc="Checkout") as pbar, flags( tqdm=pbar