From 64174b7febc3a5ab1df850fe6fa056c58311a12b Mon Sep 17 00:00:00 2001 From: Mykyta Samovarov Date: Wed, 25 Jan 2023 17:06:34 +0200 Subject: [PATCH 1/4] Initial commit --- panoply/constants.py | 2 +- panoply/datasource.py | 15 +++++++++++++-- 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/panoply/constants.py b/panoply/constants.py index a7931f6..89e1072 100644 --- a/panoply/constants.py +++ b/panoply/constants.py @@ -1,2 +1,2 @@ -__version__ = "2.0.15" +__version__ = "2.1.0" __package_name__ = "panoply-python-sdk" diff --git a/panoply/datasource.py b/panoply/datasource.py index 5e95d65..9d8c742 100644 --- a/panoply/datasource.py +++ b/panoply/datasource.py @@ -1,8 +1,10 @@ import base64 import traceback +import concurrent.futures.thread from concurrent.futures import ThreadPoolExecutor from functools import wraps from threading import Event +from time import time import backoff import requests @@ -153,7 +155,7 @@ def wrapper(*args, **kwargs): return _validate_token -def background_progress(message, waiting_interval=10 * 60): +def background_progress(message, waiting_interval=10 * 60, timeout=None): """ A decorator is used to emit progress while long operation is executed. For example, for database's data sources such operations might be declaration of the cursor or counting number of rows. @@ -167,6 +169,9 @@ def background_progress(message, waiting_interval=10 * 60): waiting_interval : float Time in seconds to wait between progress emitting. Defaults to 10 minutes + timeout : float + Time in seconds for maximum progress emiting time. + Defaults to no limit """ def _background_progress(func): @@ -175,11 +180,17 @@ def wrapper(*args, **kwargs): self = args[0] self.log('Creating background progress emitter') finished = Event() + started_at = time() with ThreadPoolExecutor(max_workers=1) as executor: func_future = executor.submit(func, *args, **kwargs) func_future.add_done_callback(lambda future: finished.set()) while not func_future.done(): + if timeout and (time() - started_at) > timeout: + self.log("Max waiting time exceeded") + executor._threads.clear() + concurrent.futures.thread._threads_queues.clear() + raise Exception("Max waiting time exceeded") self.log(message) self.progress(None, None, message) finished.wait(timeout=waiting_interval) @@ -188,4 +199,4 @@ def wrapper(*args, **kwargs): return wrapper - return _background_progress + return _background_progress \ No newline at end of file From 77bc6f9b5ce8361cd96bbe5e12e32d6f2395d34f Mon Sep 17 00:00:00 2001 From: Mykyta Samovarov Date: Thu, 26 Jan 2023 13:33:21 +0200 Subject: [PATCH 2/4] Added new line at the end --- panoply/datasource.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/panoply/datasource.py b/panoply/datasource.py index 9d8c742..6a69667 100644 --- a/panoply/datasource.py +++ b/panoply/datasource.py @@ -199,4 +199,5 @@ def wrapper(*args, **kwargs): return wrapper - return _background_progress \ No newline at end of file + return _background_progress + From e92fb74e466665aeba93f67dd789f644c59f7dea Mon Sep 17 00:00:00 2001 From: Mykyta Samovarov Date: Thu, 26 Jan 2023 13:46:48 +0200 Subject: [PATCH 3/4] Added log; Modified log --- panoply/datasource.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/panoply/datasource.py b/panoply/datasource.py index 6a69667..8ca7386 100644 --- a/panoply/datasource.py +++ b/panoply/datasource.py @@ -179,6 +179,8 @@ def _background_progress(func): def wrapper(*args, **kwargs): self = args[0] self.log('Creating background progress emitter') + if max_wait: + self.log(f'Timeout is set to {max_wait} seconds') finished = Event() started_at = time() with ThreadPoolExecutor(max_workers=1) as executor: @@ -187,7 +189,7 @@ def wrapper(*args, **kwargs): while not func_future.done(): if timeout and (time() - started_at) > timeout: - self.log("Max waiting time exceeded") + self.log("Max waiting time exceeded. Clearing threads.") executor._threads.clear() concurrent.futures.thread._threads_queues.clear() raise Exception("Max waiting time exceeded") From 3e539ec9324befcc9cad4cd1f100cb4da56317e2 Mon Sep 17 00:00:00 2001 From: Mykyta Samovarov <83225223+MykytaPanoply@users.noreply.github.com> Date: Tue, 31 Jan 2023 20:11:36 +0200 Subject: [PATCH 4/4] Made timeout 24 hours by default --- panoply/datasource.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/panoply/datasource.py b/panoply/datasource.py index 8ca7386..50a9119 100644 --- a/panoply/datasource.py +++ b/panoply/datasource.py @@ -155,7 +155,7 @@ def wrapper(*args, **kwargs): return _validate_token -def background_progress(message, waiting_interval=10 * 60, timeout=None): +def background_progress(message, waiting_interval=10 * 60, timeout=24*60*60): """ A decorator is used to emit progress while long operation is executed. For example, for database's data sources such operations might be declaration of the cursor or counting number of rows. @@ -171,7 +171,7 @@ def background_progress(message, waiting_interval=10 * 60, timeout=None): Defaults to 10 minutes timeout : float Time in seconds for maximum progress emiting time. - Defaults to no limit + Defaults to no 24 hours """ def _background_progress(func): @@ -179,8 +179,7 @@ def _background_progress(func): def wrapper(*args, **kwargs): self = args[0] self.log('Creating background progress emitter') - if max_wait: - self.log(f'Timeout is set to {max_wait} seconds') + self.log(f'Timeout is set to {max_wait} seconds') finished = Event() started_at = time() with ThreadPoolExecutor(max_workers=1) as executor: @@ -188,7 +187,7 @@ def wrapper(*args, **kwargs): func_future.add_done_callback(lambda future: finished.set()) while not func_future.done(): - if timeout and (time() - started_at) > timeout: + if (time() - started_at) > timeout: self.log("Max waiting time exceeded. Clearing threads.") executor._threads.clear() concurrent.futures.thread._threads_queues.clear()