From ed3352f12355d20353e9284ceed282b96736b55c Mon Sep 17 00:00:00 2001 From: zoriana2021 Date: Mon, 14 Nov 2022 15:41:17 +0200 Subject: [PATCH 01/16] updated paramico version --- panoply/constants.py | 2 +- setup.py | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/panoply/constants.py b/panoply/constants.py index ad33c2e..a7931f6 100644 --- a/panoply/constants.py +++ b/panoply/constants.py @@ -1,2 +1,2 @@ -__version__ = "2.0.14" +__version__ = "2.0.15" __package_name__ = "panoply-python-sdk" diff --git a/setup.py b/setup.py index ebc0d84..3807ac8 100644 --- a/setup.py +++ b/setup.py @@ -21,8 +21,7 @@ "oauth2client==4.1.1", "backoff==1.10.0", "sshtunnel==0.1.5", - "paramiko==2.7.2", - "cryptography==36.0.2", + "paramiko==2.11.0", ], extras_require={ "test": [ From 64174b7febc3a5ab1df850fe6fa056c58311a12b Mon Sep 17 00:00:00 2001 From: Mykyta Samovarov Date: Wed, 25 Jan 2023 17:06:34 +0200 Subject: [PATCH 02/16] Initial commit --- panoply/constants.py | 2 +- panoply/datasource.py | 15 +++++++++++++-- 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/panoply/constants.py b/panoply/constants.py index a7931f6..89e1072 100644 --- a/panoply/constants.py +++ b/panoply/constants.py @@ -1,2 +1,2 @@ -__version__ = "2.0.15" +__version__ = "2.1.0" __package_name__ = "panoply-python-sdk" diff --git a/panoply/datasource.py b/panoply/datasource.py index 5e95d65..9d8c742 100644 --- a/panoply/datasource.py +++ b/panoply/datasource.py @@ -1,8 +1,10 @@ import base64 import traceback +import concurrent.futures.thread from concurrent.futures import ThreadPoolExecutor from functools import wraps from threading import Event +from time import time import backoff import requests @@ -153,7 +155,7 @@ def wrapper(*args, **kwargs): return _validate_token -def background_progress(message, waiting_interval=10 * 60): +def background_progress(message, waiting_interval=10 * 60, timeout=None): """ A decorator is used to emit progress while long operation is executed. For example, for database's data sources such operations might be declaration of the cursor or counting number of rows. @@ -167,6 +169,9 @@ def background_progress(message, waiting_interval=10 * 60): waiting_interval : float Time in seconds to wait between progress emitting. Defaults to 10 minutes + timeout : float + Time in seconds for maximum progress emiting time. + Defaults to no limit """ def _background_progress(func): @@ -175,11 +180,17 @@ def wrapper(*args, **kwargs): self = args[0] self.log('Creating background progress emitter') finished = Event() + started_at = time() with ThreadPoolExecutor(max_workers=1) as executor: func_future = executor.submit(func, *args, **kwargs) func_future.add_done_callback(lambda future: finished.set()) while not func_future.done(): + if timeout and (time() - started_at) > timeout: + self.log("Max waiting time exceeded") + executor._threads.clear() + concurrent.futures.thread._threads_queues.clear() + raise Exception("Max waiting time exceeded") self.log(message) self.progress(None, None, message) finished.wait(timeout=waiting_interval) @@ -188,4 +199,4 @@ def wrapper(*args, **kwargs): return wrapper - return _background_progress + return _background_progress \ No newline at end of file From 77bc6f9b5ce8361cd96bbe5e12e32d6f2395d34f Mon Sep 17 00:00:00 2001 From: Mykyta Samovarov Date: Thu, 26 Jan 2023 13:33:21 +0200 Subject: [PATCH 03/16] Added new line at the end --- panoply/datasource.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/panoply/datasource.py b/panoply/datasource.py index 9d8c742..6a69667 100644 --- a/panoply/datasource.py +++ b/panoply/datasource.py @@ -199,4 +199,5 @@ def wrapper(*args, **kwargs): return wrapper - return _background_progress \ No newline at end of file + return _background_progress + From e92fb74e466665aeba93f67dd789f644c59f7dea Mon Sep 17 00:00:00 2001 From: Mykyta Samovarov Date: Thu, 26 Jan 2023 13:46:48 +0200 Subject: [PATCH 04/16] Added log; Modified log --- panoply/datasource.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/panoply/datasource.py b/panoply/datasource.py index 6a69667..8ca7386 100644 --- a/panoply/datasource.py +++ b/panoply/datasource.py @@ -179,6 +179,8 @@ def _background_progress(func): def wrapper(*args, **kwargs): self = args[0] self.log('Creating background progress emitter') + if max_wait: + self.log(f'Timeout is set to {max_wait} seconds') finished = Event() started_at = time() with ThreadPoolExecutor(max_workers=1) as executor: @@ -187,7 +189,7 @@ def wrapper(*args, **kwargs): while not func_future.done(): if timeout and (time() - started_at) > timeout: - self.log("Max waiting time exceeded") + self.log("Max waiting time exceeded. Clearing threads.") executor._threads.clear() concurrent.futures.thread._threads_queues.clear() raise Exception("Max waiting time exceeded") From 3e539ec9324befcc9cad4cd1f100cb4da56317e2 Mon Sep 17 00:00:00 2001 From: Mykyta Samovarov <83225223+MykytaPanoply@users.noreply.github.com> Date: Tue, 31 Jan 2023 20:11:36 +0200 Subject: [PATCH 05/16] Made timeout 24 hours by default --- panoply/datasource.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/panoply/datasource.py b/panoply/datasource.py index 8ca7386..50a9119 100644 --- a/panoply/datasource.py +++ b/panoply/datasource.py @@ -155,7 +155,7 @@ def wrapper(*args, **kwargs): return _validate_token -def background_progress(message, waiting_interval=10 * 60, timeout=None): +def background_progress(message, waiting_interval=10 * 60, timeout=24*60*60): """ A decorator is used to emit progress while long operation is executed. For example, for database's data sources such operations might be declaration of the cursor or counting number of rows. @@ -171,7 +171,7 @@ def background_progress(message, waiting_interval=10 * 60, timeout=None): Defaults to 10 minutes timeout : float Time in seconds for maximum progress emiting time. - Defaults to no limit + Defaults to no 24 hours """ def _background_progress(func): @@ -179,8 +179,7 @@ def _background_progress(func): def wrapper(*args, **kwargs): self = args[0] self.log('Creating background progress emitter') - if max_wait: - self.log(f'Timeout is set to {max_wait} seconds') + self.log(f'Timeout is set to {max_wait} seconds') finished = Event() started_at = time() with ThreadPoolExecutor(max_workers=1) as executor: @@ -188,7 +187,7 @@ def wrapper(*args, **kwargs): func_future.add_done_callback(lambda future: finished.set()) while not func_future.done(): - if timeout and (time() - started_at) > timeout: + if (time() - started_at) > timeout: self.log("Max waiting time exceeded. Clearing threads.") executor._threads.clear() concurrent.futures.thread._threads_queues.clear() From 376f01179e8ab6139cd7034d66a8f0dc8c4af1bc Mon Sep 17 00:00:00 2001 From: Mykyta Samovarov <83225223+MykytaPanoply@users.noreply.github.com> Date: Wed, 1 Feb 2023 14:08:29 +0200 Subject: [PATCH 06/16] Fixed error with renaming variable --- panoply/datasource.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/panoply/datasource.py b/panoply/datasource.py index 50a9119..7c6bab4 100644 --- a/panoply/datasource.py +++ b/panoply/datasource.py @@ -179,7 +179,7 @@ def _background_progress(func): def wrapper(*args, **kwargs): self = args[0] self.log('Creating background progress emitter') - self.log(f'Timeout is set to {max_wait} seconds') + self.log(f'Timeout is set to {timeout} seconds') finished = Event() started_at = time() with ThreadPoolExecutor(max_workers=1) as executor: From 8db1db19c3b3e07cf66fdaad8d8d19b8def021f3 Mon Sep 17 00:00:00 2001 From: Danylo Date: Wed, 15 Mar 2023 12:15:32 +0200 Subject: [PATCH 07/16] added logs for invalid token --- panoply/constants.py | 2 +- panoply/datasource.py | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/panoply/constants.py b/panoply/constants.py index 89e1072..27f5e63 100644 --- a/panoply/constants.py +++ b/panoply/constants.py @@ -1,2 +1,2 @@ -__version__ = "2.1.0" +__version__ = "2.1.1" __package_name__ = "panoply-python-sdk" diff --git a/panoply/datasource.py b/panoply/datasource.py index 7c6bab4..5dfad6a 100644 --- a/panoply/datasource.py +++ b/panoply/datasource.py @@ -139,6 +139,9 @@ def wrapper(*args, **kwargs): _callback = getattr(self, callback) _callback(self.source.get(access_key)) except Exception as e: + response = getattr(e, 'response', None) + if isinstance(response, requests.Response): + self.log(response.text) self.log('Error: Access token can\'t be revalidated. ' 'The user would have to re-authenticate', traceback.format_exc()) From 373aa5c90f5a08e848dd9de63cafc927adc60018 Mon Sep 17 00:00:00 2001 From: Mykyta Samovarov Date: Thu, 23 Mar 2023 16:08:56 +0200 Subject: [PATCH 08/16] Added STDOUTONLY logger into SSH --- panoply/constants.py | 2 +- panoply/ssh.py | 12 +++++++++++- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/panoply/constants.py b/panoply/constants.py index a7931f6..e7d4829 100644 --- a/panoply/constants.py +++ b/panoply/constants.py @@ -1,2 +1,2 @@ -__version__ = "2.0.15" +__version__ = "2.0.16" __package_name__ = "panoply-python-sdk" diff --git a/panoply/ssh.py b/panoply/ssh.py index 1c843ac..b714391 100644 --- a/panoply/ssh.py +++ b/panoply/ssh.py @@ -1,14 +1,23 @@ """ Module for storing SSH related stuff """ +import logging from typing import Dict from paramiko import RSAKey, Ed25519Key, SSHException from sshtunnel import SSHTunnelForwarder from io import StringIO +from sys import stdout from .errors import IncorrectParamError +def get_stdout_only_logger(): + logger = logging.getLogger("STDOUTONLY") + stream_handler = logging.StreamHandler(stream=stdout) + logger.addHandler(stream_handler) + return logger + + class SSHTunnel: """ General SSH tunnel class-component @@ -144,7 +153,8 @@ def _get_server(self, platform_ssh: bool): ssh_username=self.tunnel["username"], ssh_password=self.tunnel.get("password"), ssh_pkey=pkey, - remote_bind_address=(self.host, self.port) + remote_bind_address=(self.host, self.port), + logger=get_stdout_only_logger() ) server.start() From 4a5cfd10852c876ce096ee30ef31b2d67f58d388 Mon Sep 17 00:00:00 2001 From: andrey-panoply Date: Tue, 16 May 2023 17:49:14 +0300 Subject: [PATCH 09/16] Print threading errors to stdout --- panoply/__init__.py | 25 +++++++++++++++++++++++++ panoply/constants.py | 2 +- 2 files changed, 26 insertions(+), 1 deletion(-) diff --git a/panoply/__init__.py b/panoply/__init__.py index 94888b0..f19b854 100644 --- a/panoply/__init__.py +++ b/panoply/__init__.py @@ -1,3 +1,28 @@ +from sys import stdout +from traceback import print_exception as _print_exception + from .datasource import * from .sdk import * from .ssh import SSHTunnel + + +def custom_excepthook(args, /): + """ + Handle uncaught Thread.run() exception + and print error text to STDOUT instead of STDERR. + + "It's always assumed that + the runnner is single-threaded and synchronuous such that `result` events + are only assigned to the last executed request". + see https://github.com/panoplyio/legacy-source-wrapper/blob/master/src/sources-runner/index.js#L74 + """ + if args.exc_type == SystemExit: + # silently ignore SystemExit + return + + print(f"Caught an exception in thread:") + _print_exception(args.exc_type, args.exc_value, args.exc_traceback, + file=stdout) + + +threading.excepthook = custom_excepthook diff --git a/panoply/constants.py b/panoply/constants.py index badb4ab..9cf51c9 100644 --- a/panoply/constants.py +++ b/panoply/constants.py @@ -1,2 +1,2 @@ -__version__ = "2.1.2" +__version__ = "2.1.3" __package_name__ = "panoply-python-sdk" From e683a5642adf2f270fff3588ac4dad1fc64ed285 Mon Sep 17 00:00:00 2001 From: andrey-panoply Date: Mon, 22 May 2023 12:04:40 +0300 Subject: [PATCH 10/16] Add console logger with STDOUT stream --- panoply/__init__.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/panoply/__init__.py b/panoply/__init__.py index f19b854..b3e243c 100644 --- a/panoply/__init__.py +++ b/panoply/__init__.py @@ -26,3 +26,8 @@ def custom_excepthook(args, /): threading.excepthook = custom_excepthook + +# create console logger and log messages to STDOUT (default stream is STDERR) +logger = logging.getLogger(__name__) +console_handler = logging.StreamHandler(stream=stdout) +logger.addHandler(console_handler) From 92597c1387d9c287a55c26f2d78bde5eced8f5d2 Mon Sep 17 00:00:00 2001 From: andrey-panoply Date: Mon, 22 May 2023 12:21:03 +0300 Subject: [PATCH 11/16] Bump version to 2.2.0 --- panoply/constants.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/panoply/constants.py b/panoply/constants.py index 9cf51c9..9b509c2 100644 --- a/panoply/constants.py +++ b/panoply/constants.py @@ -1,2 +1,2 @@ -__version__ = "2.1.3" +__version__ = "2.2.0" __package_name__ = "panoply-python-sdk" From aebade4b76fd15689bb7e55fe8e71bba9fb513b1 Mon Sep 17 00:00:00 2001 From: andrey-panoply Date: Mon, 22 May 2023 15:42:07 +0300 Subject: [PATCH 12/16] Change logger --- panoply/__init__.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/panoply/__init__.py b/panoply/__init__.py index b3e243c..5f75e77 100644 --- a/panoply/__init__.py +++ b/panoply/__init__.py @@ -1,3 +1,5 @@ +import logging + from sys import stdout from traceback import print_exception as _print_exception @@ -6,6 +8,9 @@ from .ssh import SSHTunnel +logging.basicConfig(stream=stdout) + + def custom_excepthook(args, /): """ Handle uncaught Thread.run() exception @@ -20,14 +25,9 @@ def custom_excepthook(args, /): # silently ignore SystemExit return - print(f"Caught an exception in thread:") + logging.error(f"Caught an exception in thread:") _print_exception(args.exc_type, args.exc_value, args.exc_traceback, file=stdout) threading.excepthook = custom_excepthook - -# create console logger and log messages to STDOUT (default stream is STDERR) -logger = logging.getLogger(__name__) -console_handler = logging.StreamHandler(stream=stdout) -logger.addHandler(console_handler) From 3ebff190f4fcf0b9c31fb66cb644154451879c1c Mon Sep 17 00:00:00 2001 From: andrey-panoply Date: Mon, 22 May 2023 16:12:20 +0300 Subject: [PATCH 13/16] Add logging format --- panoply/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/panoply/__init__.py b/panoply/__init__.py index 5f75e77..91ecc03 100644 --- a/panoply/__init__.py +++ b/panoply/__init__.py @@ -8,7 +8,7 @@ from .ssh import SSHTunnel -logging.basicConfig(stream=stdout) +logging.basicConfig(stream=stdout, format='%(levelname)s: %(message)s') def custom_excepthook(args, /): From 11194f0540e20bf85e83a9c72deefac81369b81d Mon Sep 17 00:00:00 2001 From: andrey-panoply Date: Mon, 22 Jul 2024 15:03:04 +0400 Subject: [PATCH 14/16] Hardcode "cryptography == 42.0.8", --- setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.py b/setup.py index 3807ac8..194d62c 100644 --- a/setup.py +++ b/setup.py @@ -22,6 +22,7 @@ "backoff==1.10.0", "sshtunnel==0.1.5", "paramiko==2.11.0", + "cryptography == 42.0.8", ], extras_require={ "test": [ From eeb40e164765b397ffe37644c40cc85861e68d3d Mon Sep 17 00:00:00 2001 From: andrey-panoply Date: Mon, 22 Jul 2024 15:03:14 +0400 Subject: [PATCH 15/16] Bump version to 2.2.1 --- panoply/constants.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/panoply/constants.py b/panoply/constants.py index 9b509c2..a937efc 100644 --- a/panoply/constants.py +++ b/panoply/constants.py @@ -1,2 +1,2 @@ -__version__ = "2.2.0" +__version__ = "2.2.1" __package_name__ = "panoply-python-sdk" From 74c8ed817dd8a76644e55adf4d5e1f1f436f3e7a Mon Sep 17 00:00:00 2001 From: Kirill Date: Mon, 14 Oct 2024 10:53:19 +0200 Subject: [PATCH 16/16] Remove unused imports --- panoply/__init__.py | 6 ++---- panoply/resources.py | 2 +- panoply/sdk.py | 9 ++++----- 3 files changed, 7 insertions(+), 10 deletions(-) diff --git a/panoply/__init__.py b/panoply/__init__.py index 68f9504..4587a3f 100644 --- a/panoply/__init__.py +++ b/panoply/__init__.py @@ -1,5 +1,4 @@ import logging - from sys import stdout from traceback import print_exception as _print_exception @@ -9,11 +8,10 @@ from .sdk import * from .ssh import SSHTunnel - logging.basicConfig(stream=stdout, format='%(levelname)s: %(message)s') -def custom_excepthook(args, /): +def custom_excepthook(args): """ Handle uncaught Thread.run() exception and print error text to STDOUT instead of STDERR. @@ -27,7 +25,7 @@ def custom_excepthook(args, /): # silently ignore SystemExit return - logging.error(f"Caught an exception in thread:") + logging.error("Caught an exception in thread:") _print_exception(args.exc_type, args.exc_value, args.exc_traceback, file=stdout) diff --git a/panoply/resources.py b/panoply/resources.py index eb651ff..f939a64 100644 --- a/panoply/resources.py +++ b/panoply/resources.py @@ -1,4 +1,4 @@ -from typing import Optional, List, TypedDict, Dict +from typing import List, Optional, TypedDict class Field(TypedDict): diff --git a/panoply/sdk.py b/panoply/sdk.py index d9dc250..0f81b74 100644 --- a/panoply/sdk.py +++ b/panoply/sdk.py @@ -1,16 +1,15 @@ import base64 import json +import queue +import threading import time -import urllib.request import urllib.error import urllib.parse -import threading -import queue -import logging +import urllib.request from copy import copy -from .constants import __package_name__, __version__ from . import events +from .constants import __package_name__, __version__ MAXSIZE = 1024 * 250 # 250kib FLUSH_TIMEOUT = 2.0 # 2 seconds