From 6fdba1dddf661e4811de1ef0c4e5e1f18d0ea9f3 Mon Sep 17 00:00:00 2001 From: Casper da Costa-Luis Date: Sun, 17 Nov 2019 22:58:47 +0000 Subject: [PATCH] initial upload idea - fixes #1566 --- dvc/remote/gs.py | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/dvc/remote/gs.py b/dvc/remote/gs.py index c2fb4efc1c..7d7eefe9fd 100644 --- a/dvc/remote/gs.py +++ b/dvc/remote/gs.py @@ -3,12 +3,15 @@ import logging from datetime import timedelta from functools import wraps +import io + from funcy import cached_property from dvc.config import Config from dvc.exceptions import DvcException from dvc.path_info import CloudURLInfo +from dvc.progress import Tqdm from dvc.remote.base import RemoteBASE from dvc.scheme import Schemes from dvc.utils.compat import FileNotFoundError # skipcq: PYL-W0622 @@ -46,9 +49,20 @@ def wrapper(*args, **kwargs): @dynamic_chunk_size -def _upload_to_bucket(bucket, from_file, to_info, **kwargs): - blob = bucket.blob(to_info.path, **kwargs) - blob.upload_from_filename(from_file) +def _upload_to_bucket(bucket, from_file, to_info, chunk_size=None, **kwargs): + blob = bucket.blob(to_info.path, chunk_size=chunk_size, **kwargs) + with Tqdm() as pbar: + with io.open(from_file, mode="rb", buffering=chunk_size or -1) as fd: + raw_read = fd.read + + def read(self, size=chunk_size): + res = raw_read(size) + if res: + pbar.update(len(res)) + return res + + fd.read = read + blob.upload_from_file(fd) class RemoteGS(RemoteBASE):