From 80c9f8ba03b398ca0b132fd8c5d3dec63549d9ef Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miguel=20Gonz=C3=A1lez-Fierro?= <3491412+miguelgfierro@users.noreply.github.com> Date: Tue, 1 Jun 2021 15:37:46 +0100 Subject: [PATCH 1/3] Remove html files in docs from being pushed --- .gitignore | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 2e60f10959..78eaf9cdee 100644 --- a/.gitignore +++ b/.gitignore @@ -155,6 +155,14 @@ ml-20m/ *.model *.mml nohup.out +*.svg +*.html +*.js +*.css +*.tff +*.woff +*.woff2 +*.eot ##### kdd 2020 tutorial data folder examples/07_tutorials/KDD2020-tutorial/data_folder/ @@ -164,4 +172,4 @@ examples/07_tutorials/KDD2020-tutorial/data_folder/ *.sh tests/**/resources/ -reports/ \ No newline at end of file +reports/ From b2c8bc0cb5546d263082b1d7f86aa7ccafebf959 Mon Sep 17 00:00:00 2001 From: miguelgfierro Date: Tue, 1 Jun 2021 18:29:06 +0000 Subject: [PATCH 2/3] backoff --- reco_utils/dataset/download_utils.py | 13 ++++++++++--- setup.py | 1 + 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/reco_utils/dataset/download_utils.py b/reco_utils/dataset/download_utils.py index 0d0d19d6ad..2f36cb7a85 100644 --- a/reco_utils/dataset/download_utils.py +++ b/reco_utils/dataset/download_utils.py @@ -9,10 +9,17 @@ from contextlib import contextmanager from tempfile import TemporaryDirectory from tqdm import tqdm +import backoff + log = logging.getLogger(__name__) +@backoff.on_exception( + backoff.expo, + (requests.exceptions.HTTPError, requests.exceptions.ChunkedEncodingError), + max_tries=5, +) def maybe_download(url, filename=None, work_directory=".", expected_bytes=None): """Download a file if it is not already downloaded. @@ -21,7 +28,7 @@ def maybe_download(url, filename=None, work_directory=".", expected_bytes=None): work_directory (str): Working directory. url (str): URL of the file to download. expected_bytes (int): Expected file size in bytes. - + Returns: str: File path of the file downloaded. """ @@ -57,8 +64,8 @@ def maybe_download(url, filename=None, work_directory=".", expected_bytes=None): @contextmanager def download_path(path=None): - """Return a path to download data. If `path=None`, then it yields a temporal path that is eventually deleted, - otherwise the real path of the input. + """Return a path to download data. If `path=None`, then it yields a temporal path that is eventually deleted, + otherwise the real path of the input. Args: path (str): Path to download data. diff --git a/setup.py b/setup.py index 8fe269ec01..7665494f9e 100644 --- a/setup.py +++ b/setup.py @@ -47,6 +47,7 @@ "tqdm>=4.31.1,<5", "transformers>=2.5.0,<5", "xlearn==0.40a1", + "backoff>=1.8.0", ] # shared dependencies From a59636411e3e35a2d5d63e01d28f00d78e9417b6 Mon Sep 17 00:00:00 2001 From: miguelgfierro Date: Tue, 1 Jun 2021 20:33:53 +0000 Subject: [PATCH 3/3] :memo: --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 7665494f9e..241bc01907 100644 --- a/setup.py +++ b/setup.py @@ -23,6 +23,7 @@ name = environ.get("LIBRARY_NAME", "reco_utils") install_requires = [ + "backoff>=1.8.0", "bottleneck>=1.2.1,<2", "category_encoders>=1.3.0,<2", "cornac>=1.11.0,<2", @@ -47,7 +48,6 @@ "tqdm>=4.31.1,<5", "transformers>=2.5.0,<5", "xlearn==0.40a1", - "backoff>=1.8.0", ] # shared dependencies