From 908495d348c8a330933aa01223d9e0a91b3ae97f Mon Sep 17 00:00:00 2001 From: Casper da Costa-Luis Date: Sun, 17 Nov 2019 22:58:47 +0000 Subject: [PATCH 1/2] initial upload idea - fixes #1566 --- dvc/remote/gs.py | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/dvc/remote/gs.py b/dvc/remote/gs.py index c2fb4efc1c..d430c3bf9f 100644 --- a/dvc/remote/gs.py +++ b/dvc/remote/gs.py @@ -3,12 +3,14 @@ import logging from datetime import timedelta from functools import wraps +import io from funcy import cached_property from dvc.config import Config from dvc.exceptions import DvcException from dvc.path_info import CloudURLInfo +from dvc.progress import Tqdm from dvc.remote.base import RemoteBASE from dvc.scheme import Schemes from dvc.utils.compat import FileNotFoundError # skipcq: PYL-W0622 @@ -46,9 +48,18 @@ def wrapper(*args, **kwargs): @dynamic_chunk_size -def _upload_to_bucket(bucket, from_file, to_info, **kwargs): - blob = bucket.blob(to_info.path, **kwargs) - blob.upload_from_filename(from_file) +def _upload_to_bucket(bucket, from_file, to_info, chunk_size=None, **kwargs): + blob = bucket.blob(to_info.path, chunk_size=chunk_size, **kwargs) + with Tqdm() as pbar: + with io.open(from_file, mode="rb", buffering=chunk_size or -1) as fd: + raw_read = fd.read + def read(self, size=chunk_size): + res = raw_read(size) + if res: + pbar.update(len(res)) + return res + fd.read = read + blob.upload_from_file(fd) class RemoteGS(RemoteBASE): From daadd77def5344809f509073ea423e38877abff4 Mon Sep 17 00:00:00 2001 From: "Restyled.io" Date: Sun, 17 Nov 2019 23:04:59 +0000 Subject: [PATCH 2/2] Restyled by black --- dvc/remote/gs.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/dvc/remote/gs.py b/dvc/remote/gs.py index d430c3bf9f..3c77de6ea7 100644 --- a/dvc/remote/gs.py +++ b/dvc/remote/gs.py @@ -53,11 +53,13 @@ def _upload_to_bucket(bucket, from_file, to_info, chunk_size=None, **kwargs): with Tqdm() as pbar: with io.open(from_file, mode="rb", buffering=chunk_size or -1) as fd: raw_read = fd.read + def read(self, size=chunk_size): res = raw_read(size) if res: pbar.update(len(res)) return res + fd.read = read blob.upload_from_file(fd)