diff --git a/README.md b/README.md index 7e3f3ad3..80500593 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ Powerful Python util methods and classes that simplify common apis and tasks. -![Current Release](https://img.shields.io/badge/release-v1.34.3-blue) +![Current Release](https://img.shields.io/badge/release-v1.38.2-blue) [![codecov](https://codecov.io/gh/owasp-sbot/OSBot-Utils/graph/badge.svg?token=GNVW0COX1N)](https://codecov.io/gh/owasp-sbot/OSBot-Utils) diff --git a/osbot_utils/context_managers/capture_duration.py b/osbot_utils/context_managers/capture_duration.py index fcdc232d..d0cc063e 100644 --- a/osbot_utils/context_managers/capture_duration.py +++ b/osbot_utils/context_managers/capture_duration.py @@ -1,12 +1,13 @@ +from osbot_utils.base_classes.Type_Safe import Type_Safe from osbot_utils.utils.Misc import timestamp_utc_now -class capture_duration(): - def __init__(self): - self.duration = None - self.start_timestamp = None - self.end_timestamp = None - self.seconds = None +class capture_duration(Type_Safe): + action_name : str + duration : float + start_timestamp : int + end_timestamp : int + seconds : float def __enter__(self): self.start_timestamp = timestamp_utc_now() @@ -23,7 +24,10 @@ def data(self): def print(self): print() - print(f'action took: {self.seconds} seconds') + if self.action_name: + print(f'action "{self.action_name}" took: {self.seconds} seconds') + else: + print(f'action took: {self.seconds} seconds') class print_duration(capture_duration): diff --git a/osbot_utils/decorators/methods/cache_on_self.py b/osbot_utils/decorators/methods/cache_on_self.py index be28d569..e5f49a4c 100644 --- a/osbot_utils/decorators/methods/cache_on_self.py +++ b/osbot_utils/decorators/methods/cache_on_self.py @@ -26,6 +26,7 @@ def cache_on_self(function: T) -> T: def wrapper(*args, **kwargs): if len(args) == 0 or inspect.isclass(type(args[0])) is False: raise Exception("In Method_Wrappers.cache_on_self could not find self") + # todo: fix bug that happens when the value of reload_cache is set to False if 'reload_cache' in kwargs: # if the reload parameter is set to True reload_cache = True # set reload to True del kwargs['reload_cache'] # remove the reload parameter from the kwargs diff --git a/osbot_utils/helpers/Random_Guid.py b/osbot_utils/helpers/Random_Guid.py new file mode 100644 index 00000000..05ef70a0 --- /dev/null +++ b/osbot_utils/helpers/Random_Guid.py @@ -0,0 +1,14 @@ +# todo add to osbot utils +from osbot_utils.utils.Misc import random_guid + +class Random_Guid(str): + def __new__(cls, value=None): + if value is None: + value = random_guid() + return str.__new__(cls, value) + + def __init__(self, value=None): + self.value = value if value is not None else random_guid() + + def __str__(self): + return self diff --git a/osbot_utils/helpers/trace/Trace_Call.py b/osbot_utils/helpers/trace/Trace_Call.py index db3981e1..5e848ec0 100644 --- a/osbot_utils/helpers/trace/Trace_Call.py +++ b/osbot_utils/helpers/trace/Trace_Call.py @@ -1,5 +1,6 @@ import linecache import sys +import threading from functools import wraps from osbot_utils.base_classes.Kwargs_To_Self import Kwargs_To_Self @@ -8,6 +9,8 @@ from osbot_utils.helpers.trace.Trace_Call__Print_Lines import Trace_Call__Print_Lines from osbot_utils.helpers.trace.Trace_Call__Print_Traces import Trace_Call__Print_Traces from osbot_utils.helpers.trace.Trace_Call__View_Model import Trace_Call__View_Model +from osbot_utils.testing.Stdout import Stdout +from osbot_utils.utils.Str import ansi_to_text def trace_calls(title = None , print_traces = True , show_locals = False, source_code = False , @@ -59,6 +62,7 @@ def __init__(self, **kwargs): self.config.trace_capture_contains = self.config.trace_capture_contains or [] # and None will be quite common since we can use [] on method's params self.config.print_max_string_length = self.config.print_max_string_length or PRINT_MAX_STRING_LENGTH self.stack = self.trace_call_handler.stack + self.trace_on_thread__data = {} #self.prev_trace_function = None # Stores the previous trace function @@ -94,6 +98,13 @@ def print(self): #self.print_lines() return view_model + def print_to_str(self): + with Stdout() as stdout: + self.print() + trace_data = ansi_to_text(stdout.value()) + return trace_data + + def print_lines(self): print() view_model = self.view_data() @@ -106,6 +117,27 @@ def start(self): self.started = True # set this here so that it does show in the trace sys.settrace(self.trace_call_handler.trace_calls) # Set the new trace function + def start__on_thread(self, root_node=None): + if sys.gettrace() is None: + current_thread = threading.current_thread() + thread_sys_trace = sys.gettrace() + thread_name = current_thread.name + thread_id = current_thread.native_id + thread_nodes = [] + thread_data = dict(thread_name = thread_name , + thread_id = thread_id , + thread_nodes = thread_nodes , + thread_sys_trace = thread_sys_trace) + title = f"Thread: {thread_name} ({thread_id})" + thread_node__for_title = self.trace_call_handler.stack.add_node(title=title) # Add node with name of Thread + + if root_node: # Add node with name of node + thread_node__for_root = self.trace_call_handler.stack.add_node(root_node) + thread_nodes.append(thread_node__for_root) + thread_nodes.append(thread_node__for_title) + sys.settrace(self.trace_call_handler.trace_calls) + self.trace_on_thread__data[thread_id] = thread_data + def stop(self): if self.started: @@ -113,6 +145,18 @@ def stop(self): self.stack.empty_stack() self.started = False + def stop__on_thread(self): + current_thread = threading.current_thread() + thread_id = current_thread.native_id + thread_data = self.trace_on_thread__data.get(thread_id) + if thread_data: # if there trace_call set up in the current thread + thread_sys_trace = thread_data.get('thread_sys_trace') + thread_nodes = thread_data.get('thread_nodes') + for thread_node in thread_nodes: # remove extra nodes added during start__on_thread + self.trace_call_handler.stack.pop(thread_node) + sys.settrace(thread_sys_trace) # restore previous sys.trace value + del self.trace_on_thread__data[thread_id] + def stats(self): return self.trace_call_handler.stats diff --git a/osbot_utils/helpers/trace/Trace_Call__Config.py b/osbot_utils/helpers/trace/Trace_Call__Config.py index 36c09163..ed67810c 100644 --- a/osbot_utils/helpers/trace/Trace_Call__Config.py +++ b/osbot_utils/helpers/trace/Trace_Call__Config.py @@ -33,6 +33,7 @@ class Trace_Call__Config(Kwargs_To_Self): trace_capture_contains : list trace_enabled : bool = True trace_ignore_start_with : list + trace_ignore_contains : list trace_show_internals : bool trace_up_to_depth : int with_duration_bigger_than : float diff --git a/osbot_utils/helpers/trace/Trace_Call__Handler.py b/osbot_utils/helpers/trace/Trace_Call__Handler.py index 4115b37a..fdbc3ace 100644 --- a/osbot_utils/helpers/trace/Trace_Call__Handler.py +++ b/osbot_utils/helpers/trace/Trace_Call__Handler.py @@ -20,11 +20,19 @@ '__default__value__' , '__setattr__' , ''] -GLOBAL_MODULES_TO_IGNORE = ['osbot_utils.helpers.trace.Trace_Call' , # todo: map out and document why exactly these modules are ignore (and what is the side effect) - 'osbot_utils.helpers.CPrint' , # also see if this should be done here or at the print/view stage - 'osbot_utils.helpers.Print_Table' , - 'osbot_utils.decorators.methods.cache_on_self' , - 'codecs'] +GLOBAL_MODULES_TO_IGNORE = ['osbot_utils.helpers.trace.Trace_Call' , # todo: map out and document why exactly these modules are ignore (and what is the side effect) + 'osbot_utils.helpers.trace.Trace_Call__Config' , + 'osbot_utils.helpers.trace.Trace_Call__View_Model' , + 'osbot_utils.helpers.trace.Trace_Call__Print_Traces' , + 'osbot_utils.helpers.trace.Trace_Call__Stack' , + 'osbot_utils.base_classes.Type_Safe' , + 'osbot_utils.helpers.CPrint' , # also see if this should be done here or at the print/view stage + 'osbot_utils.helpers.Print_Table' , + 'osbot_utils.decorators.methods.cache_on_self' , + 'codecs' ] + +#GLOBAL_MODULES_TO_IGNORE = [] +#GLOBAL_FUNCTIONS_TO_IGNORE = [] class Trace_Call__Handler(Kwargs_To_Self): config : Trace_Call__Config @@ -171,6 +179,11 @@ def should_capture(self, frame): if module.startswith(item) or func_name.startswith(item): capture = False break + + for item in self.config.trace_ignore_contains: # Check if the module should be ignored + if item in module or item in func_name: + capture = False + break return capture def stack_json__parse_node(self, stack_node: Trace_Call__Stack_Node): diff --git a/osbot_utils/testing/Stderr.py b/osbot_utils/testing/Stderr.py index 6abcd471..f2af9061 100644 --- a/osbot_utils/testing/Stderr.py +++ b/osbot_utils/testing/Stderr.py @@ -2,17 +2,23 @@ from contextlib import redirect_stderr -class Stderr: +class Stderr: # todo: refactor with Stdout whose code is 90% the same as this one. Add class to capture both at the same time def __init__(self): self.output = io.StringIO() self.redirect_stderr = redirect_stderr(self.output) def __enter__(self): - self.redirect_stderr.__enter__() + self.start() return self - def __exit__(self, exc_type, exc_val, exc_tb): - self.redirect_stderr.__exit__(exc_type, exc_val, exc_tb) + def __exit__(self, *args, **kwargs): + self.stop(*args, **kwargs) + + def start(self): + self.redirect_stderr.__enter__() + + def stop(self, exc_type=None, exc_inst=None, exc_tb=None): + self.redirect_stderr.__exit__(exc_type, exc_inst, exc_tb) def value(self): return self.output.getvalue() diff --git a/osbot_utils/testing/Stdout.py b/osbot_utils/testing/Stdout.py index ee1e61db..25ffb05f 100644 --- a/osbot_utils/testing/Stdout.py +++ b/osbot_utils/testing/Stdout.py @@ -8,11 +8,17 @@ def __init__(self): self.redirect_stdout = redirect_stdout(self.output) def __enter__(self): - self.redirect_stdout.__enter__() + self.start() return self - def __exit__(self, exc_type, exc_val, exc_tb): - self.redirect_stdout.__exit__(exc_type, exc_val, exc_tb) + def __exit__(self, *args, **kwargs): + self.stop(*args, **kwargs) + + def start(self): + self.redirect_stdout.__enter__() + + def stop(self, exc_type=None, exc_inst=None, exc_tb=None): + self.redirect_stdout.__exit__(exc_type, exc_inst, exc_tb) def value(self): return self.output.getvalue() diff --git a/osbot_utils/testing/Temp_Env_Vars.py b/osbot_utils/testing/Temp_Env_Vars.py index a0b1b53d..fbe21be8 100644 --- a/osbot_utils/testing/Temp_Env_Vars.py +++ b/osbot_utils/testing/Temp_Env_Vars.py @@ -8,11 +8,18 @@ class Temp_Env_Vars(Type_Safe): original_env_vars: dict def __enter__(self): + return self.set_vars() + + def __exit__(self, exc_type, exc_value, traceback): + self.restore_vars() + + def set_vars(self): for key, value in self.env_vars.items(): self.original_env_vars[key] = os.environ.get(key) # Backup original environment variables and set new ones os.environ[key] = value + return self - def __exit__(self, exc_type, exc_value, traceback): + def restore_vars(self): for key in self.env_vars: # Restore original environment variables if self.original_env_vars[key] is None: del os.environ[key] diff --git a/osbot_utils/utils/Env.py b/osbot_utils/utils/Env.py index b7633614..15c34292 100644 --- a/osbot_utils/utils/Env.py +++ b/osbot_utils/utils/Env.py @@ -20,7 +20,9 @@ def env__pwd(): return get_env('PWD', '') def env__old_pwd__remove(value): - return value.replace(env__old_pwd(), '') + if env__old_pwd() != '/': # can't replace with old pwd is just / + return value.replace(env__old_pwd(), '') + return value def env__terminal__is__xterm(): return os.getenv('TERM') == 'xterm' @@ -91,6 +93,9 @@ def find_dotenv_file(start_path=None, env_file_to_find='.env'): def in_github_action(): return os.getenv('GITHUB_ACTIONS') == 'true' +def in_pytest_with_coverage(): + return os.getenv('COVERAGE_RUN') == 'true' + def in_python_debugger(): if sys.gettrace() is not None: # Check for a trace function return True diff --git a/osbot_utils/utils/Files.py b/osbot_utils/utils/Files.py index 26d96e5f..7a6f1173 100644 --- a/osbot_utils/utils/Files.py +++ b/osbot_utils/utils/Files.py @@ -489,6 +489,22 @@ def file_move_to_folder(source_file, target_folder): if file_move(source_file, target_file): return target_file +def files_names_without_extension(files): + return [file_name_without_extension(file) for file in files] + +def files_names_in_folder(target, with_extension=False): + if with_extension: + return files_names(files_in_folder(target)) + else: + return files_names_without_extension(files_in_folder(target)) + +def files_in_folder(path,pattern='*', only_files=True): + result = [] + for file in Path(path).glob(pattern): + if only_files and is_not_file(file): + continue + result.append(str(file)) # todo: see if there is a better way to do this conversion to string + return sorted(result) def folders_names_in_folder(target): folders = folders_in_folder(target) diff --git a/osbot_utils/utils/Lists.py b/osbot_utils/utils/Lists.py index 9297d249..4220e559 100644 --- a/osbot_utils/utils/Lists.py +++ b/osbot_utils/utils/Lists.py @@ -93,6 +93,9 @@ def list_index_by(values, index_by): def list_lower(input_list): return [item.lower() for item in input_list] +def list_minus_list(list_a, list_b): + return [item for item in list_a if item not in list_b] + def list_not_empty(list): if list and type(list).__name__ == 'list' and len(list) >0: return True diff --git a/osbot_utils/utils/Misc.py b/osbot_utils/utils/Misc.py index f026f77e..704fa6a2 100644 --- a/osbot_utils/utils/Misc.py +++ b/osbot_utils/utils/Misc.py @@ -8,6 +8,7 @@ import sys import textwrap import re +import threading import uuid import warnings from datetime import datetime, timedelta @@ -67,6 +68,10 @@ def convert_to_number(value): else: return 0 +def current_thread_id(): + return threading.current_thread().native_id + + def date_time_from_to_str(date_time_str, format_from, format_to, print_conversion_error=False): try: date_time = datetime.strptime(date_time_str, format_from) @@ -119,6 +124,9 @@ def date_time_now_less_time_delta(days=0,hours=0, minutes=0, seconds=0, date_tim def date_to_str(date, date_format='%Y-%m-%d'): return date.strftime(date_format) +def date_today(): + return date_time_now(date_time_format='%Y-%m-%d') + #note: this is here at the moment due to a circular dependency with lists and objects def list_set(target: object) -> object: if hasattr(target, '__iter__'): diff --git a/osbot_utils/version b/osbot_utils/version index 0e51036a..9a654d26 100644 --- a/osbot_utils/version +++ b/osbot_utils/version @@ -1 +1 @@ -v1.34.3 +v1.38.2 diff --git a/pyproject.toml b/pyproject.toml index aeef149d..2b176e44 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "osbot_utils" -version = "v1.34.3" +version = "v1.38.2" description = "OWASP Security Bot - Utils" authors = ["Dinis Cruz "] license = "MIT" diff --git a/tests/unit/helpers/test_Random_Guid.py b/tests/unit/helpers/test_Random_Guid.py new file mode 100644 index 00000000..58c28549 --- /dev/null +++ b/tests/unit/helpers/test_Random_Guid.py @@ -0,0 +1,28 @@ +from unittest import TestCase + +from osbot_utils.helpers.Random_Guid import Random_Guid +from osbot_utils.utils.Json import json_to_str, json_round_trip +from osbot_utils.utils.Misc import is_guid +from osbot_utils.utils.Objects import base_types + + +class test_Random_Guid(TestCase): + + def test__init__(self): + random_guid = Random_Guid() + assert len(random_guid) == 36 + assert type(random_guid) is Random_Guid + assert type(str(random_guid)) is not str # a bit weird why this is not a str + assert base_types(random_guid) == [str, object] + assert str(random_guid) == random_guid + + assert is_guid (random_guid) + assert isinstance (random_guid, str) + + assert Random_Guid() != Random_Guid() + assert str(Random_Guid()) != str(Random_Guid()) + + + assert json_to_str(random_guid) == f'"{random_guid}"' + assert json_round_trip(random_guid) == str(random_guid) + assert type(json_round_trip(random_guid)) is str diff --git a/tests/unit/helpers/trace/test_Trace_Call.py b/tests/unit/helpers/trace/test_Trace_Call.py index b2246b66..71d75a12 100644 --- a/tests/unit/helpers/trace/test_Trace_Call.py +++ b/tests/unit/helpers/trace/test_Trace_Call.py @@ -52,7 +52,8 @@ def test___init__(self): 'started' : False , 'trace_call_handler' : self.trace_call.trace_call_handler , 'trace_call_view_model' : self.trace_call.trace_call_view_model , - 'trace_call_print_traces': self.trace_call.trace_call_print_traces} + 'trace_call_print_traces': self.trace_call.trace_call_print_traces, + 'trace_on_thread__data' : {} } assert type(self.trace_call.trace_call_handler ) is Trace_Call__Handler assert type(self.trace_call.trace_call_view_model) is Trace_Call__View_Model @@ -262,8 +263,8 @@ def test__trace_up_to_a_level(self): if sys.version_info < (3, 8): pytest.skip("Skipping test that need FIXING on 3.7 or lower") - if not in_github_action(): # todo: rewrite this test to use an example that is not - return # as expensive as Python_Logger since it is taking 200+ms (which is about 50% of the all OSBot_Utils tests + if not in_github_action(): # todo: rewrite this test to use an example that is not + pytest.skip("Skipping locally since it last") # as expensive as Python_Logger since it is taking 200+ms (which is about 50% of the all OSBot_Utils tests with self.config as _: _.all() _.up_to_depth(2) @@ -275,7 +276,7 @@ def test__trace_up_to_a_level(self): 'Here are the 8 traces captured\n' , '\x1b[1m📦 Trace Session\x1b[0m' , '\x1b[1m│ ├── 🔗️ Python_Logger.__init__\x1b[0m' , - '\x1b[1m│ │ ├── 🧩️ Python_Logger.__init__\x1b[0m', + '\x1b[1m│ │ ├── 🧩️ Python_Logger_Config.__init__\x1b[0m', '\x1b[1m│ │ ├── 🧩️ set_logger_name\x1b[0m' , '\x1b[1m│ │ ├── 🧩️ set_config\x1b[0m' , '\x1b[1m│ │ └── 🧩️ setup\x1b[0m' , diff --git a/tests/unit/helpers/trace/test_Trace_Call__Config.py b/tests/unit/helpers/trace/test_Trace_Call__Config.py index 5e648606..903f3390 100644 --- a/tests/unit/helpers/trace/test_Trace_Call__Config.py +++ b/tests/unit/helpers/trace/test_Trace_Call__Config.py @@ -39,6 +39,7 @@ def test__kwargs__(self): 'trace_capture_start_with' : [] , 'trace_capture_contains' : [] , 'trace_enabled' : True , + 'trace_ignore_contains' : [] , 'trace_ignore_start_with' : [] , 'trace_show_internals' : False , 'trace_up_to_depth' : 0 , diff --git a/tests/unit/helpers/trace/test_Trace_Files.py b/tests/unit/helpers/trace/test_Trace_Files.py index cb1a33eb..341ab958 100644 --- a/tests/unit/helpers/trace/test_Trace_Files.py +++ b/tests/unit/helpers/trace/test_Trace_Files.py @@ -21,7 +21,8 @@ def test___default_kwargs__(self): 'stack' : trace_files.stack , 'trace_call_handler' : trace_files.trace_call_handler , 'trace_call_print_traces' : trace_files.trace_call_print_traces, - 'trace_call_view_model' : trace_files.trace_call_view_model } + 'trace_call_view_model' : trace_files.trace_call_view_model , + 'trace_on_thread__data' : {} } trace_files.stack.add_node(DEFAULT_ROOT_NODE_NODE_TITLE) assert len(trace_files.stack) == 1