diff --git a/docs/docs_config.py b/docs/docs_config.py index 38d8165b9..8df157484 100644 --- a/docs/docs_config.py +++ b/docs/docs_config.py @@ -40,7 +40,7 @@ import lbaf.Utils.csv_2_json_data_converter as Csv2JsonConverter import lbaf.Utils.data_stat_files_updater as DataStatFilesUpdater import lbaf.Utils.exception_handler as exc_handler -import lbaf.Utils.logging as logging +import lbaf.Utils.logger as logger import lbaf.Utils.vt_data_extractor as VTDataExtractor @@ -97,7 +97,7 @@ Csv2JsonConverter, DataStatFilesUpdater, exc_handler, - logging, + logger, VTDataExtractor, configurationUpgrader] diff --git a/scripts/upgrade_configuration.py b/scripts/upgrade_configuration.py index f52fa9643..3f15d32ee 100644 --- a/scripts/upgrade_configuration.py +++ b/scripts/upgrade_configuration.py @@ -2,7 +2,7 @@ import argparse from lbaf import PROJECT_PATH -from lbaf.Utils.logging import get_logger +from lbaf.Utils.logger import get_logger from lbaf.IO.lbsConfigurationUpgrader import ConfigurationUpgrader, UpgradeAction @@ -27,8 +27,15 @@ ) args = parser.parse_args() -if not args.add and not args.remove: - raise ValueError("Missing either add (-a or --add xxx) or remove arg (-r or --remove xxx)") +ACTION = UpgradeAction.FORMAT_ONLY +KEY = None +if args.add: + ACTION = UpgradeAction.ADD_KEY + KEY = args.add +elif args.remove: + ACTION = UpgradeAction.REMOVE_KEY + KEY = args.remove + if args.add and args.remove: raise ValueError("Cannot set both add and remove args") if args.add and not args.value: @@ -45,9 +52,9 @@ upgrader = ConfigurationUpgrader(logger=upg_logger) upgrader.upgrade_all( pattern = args.pattern, - relative_to = PROJECT_PATH, - action=(UpgradeAction.ADD_KEY if args.add is not None else UpgradeAction.REMOVE_KEY), - key=(args.add if args.add is not None else args.remove), + relative_to=PROJECT_PATH, + action=ACTION, + key=KEY, value=args.value, value_type=args.type ) diff --git a/somefile b/somefile deleted file mode 100644 index 9902f1784..000000000 --- a/somefile +++ /dev/null @@ -1 +0,0 @@ -28 diff --git a/src/lbaf/Applications/JSON_data_files_validator_loader.py b/src/lbaf/Applications/JSON_data_files_validator_loader.py index 212f63f5e..e461860a8 100644 --- a/src/lbaf/Applications/JSON_data_files_validator_loader.py +++ b/src/lbaf/Applications/JSON_data_files_validator_loader.py @@ -5,13 +5,15 @@ from urllib.request import urlretrieve from urllib.error import HTTPError, URLError -from .. import PROJECT_PATH -from ..Utils.logging import get_logger -from ..Utils.exception_handler import exc_handler +from lbaf import PROJECT_PATH +from lbaf.Utils.logger import get_logger +from lbaf.Utils.exception_handler import exc_handler TARGET_DIR = os.path.join(PROJECT_PATH, "src", "lbaf", "imported") TARGET_SCRIPT_NAME = "JSON_data_files_validator.py" SOURCE_SCRIPT_URL=f"https://raw.githubusercontent.com/DARMA-tasking/vt/develop/scripts/{TARGET_SCRIPT_NAME}" +IMPORT_DIR = os.path.join(PROJECT_PATH, "src", "lbaf", "imported") + def _save_schema_validator_and_init_file(): """Downloads the JSON data files validator module to the lbaf/imported directory. @@ -80,5 +82,7 @@ def is_loaded(): :return: True if the script exists in the target location otherwise False :rtype: bool """ - import_dir = os.path.join(PROJECT_PATH, "src", "lbaf", "imported") - return os.path.isfile(os.path.join(import_dir, TARGET_SCRIPT_NAME)) + return os.path.isfile(os.path.join(IMPORT_DIR, TARGET_SCRIPT_NAME)) + +if __name__ == "__main__": + load() diff --git a/src/lbaf/Applications/LBAF_app.py b/src/lbaf/Applications/LBAF_app.py index 838a91926..366de1f66 100644 --- a/src/lbaf/Applications/LBAF_app.py +++ b/src/lbaf/Applications/LBAF_app.py @@ -13,7 +13,7 @@ from lbaf.Applications import JSON_data_files_validator_loader from lbaf.Utils.exception_handler import exc_handler from lbaf.Utils.path import abspath -from lbaf.Utils.logging import get_logger, Logger +from lbaf.Utils.logger import get_logger, Logger import lbaf.IO.lbsStatistics as lbstats from lbaf.IO.lbsConfigurationValidator import ConfigurationValidator from lbaf.IO.lbsVTDataReader import LoadReader diff --git a/src/lbaf/Applications/MoveCountsViewer.py b/src/lbaf/Applications/MoveCountsViewer.py index 073be43f5..293b2a617 100644 --- a/src/lbaf/Applications/MoveCountsViewer.py +++ b/src/lbaf/Applications/MoveCountsViewer.py @@ -7,7 +7,7 @@ from lbaf import PROJECT_PATH from lbaf.Utils.exception_handler import exc_handler -from lbaf.Utils.logging import get_logger, Logger +from lbaf.Utils.logger import get_logger, Logger class MoveCountsViewerParameters: diff --git a/src/lbaf/Applications/rank_object_enumerator.py b/src/lbaf/Applications/rank_object_enumerator.py index 7c5e097c5..ea1d1a570 100644 --- a/src/lbaf/Applications/rank_object_enumerator.py +++ b/src/lbaf/Applications/rank_object_enumerator.py @@ -8,7 +8,7 @@ from lbaf import PROJECT_PATH from lbaf.IO.lbsVTDataReader import LoadReader from lbaf.Utils.exception_handler import exc_handler -from lbaf.Utils.logging import get_logger +from lbaf.Utils.logger import get_logger from lbaf.Utils.path import abspath from lbaf.Model.lbsPhase import Phase from lbaf.IO.lbsStatistics import ( diff --git a/src/lbaf/Execution/lbsAlgorithmBase.py b/src/lbaf/Execution/lbsAlgorithmBase.py index 5e37c994a..be95dbea9 100644 --- a/src/lbaf/Execution/lbsAlgorithmBase.py +++ b/src/lbaf/Execution/lbsAlgorithmBase.py @@ -6,7 +6,7 @@ from ..Model.lbsPhase import Phase from ..Model.lbsWorkModelBase import WorkModelBase from ..Utils.exception_handler import exc_handler -from ..Utils.logging import get_logger, Logger +from ..Utils.logger import get_logger, Logger class AlgorithmBase: diff --git a/src/lbaf/Execution/lbsCriterionBase.py b/src/lbaf/Execution/lbsCriterionBase.py index e1611a4a3..ac2401bdc 100644 --- a/src/lbaf/Execution/lbsCriterionBase.py +++ b/src/lbaf/Execution/lbsCriterionBase.py @@ -6,7 +6,6 @@ from ..Model.lbsWorkModelBase import WorkModelBase from ..Model.lbsPhase import Phase from ..Utils.exception_handler import exc_handler -from ..Utils.logging import get_logger class CriterionBase: diff --git a/src/lbaf/IO/lbsConfigurationUpgrader.py b/src/lbaf/IO/lbsConfigurationUpgrader.py index e9379258e..e926c1630 100644 --- a/src/lbaf/IO/lbsConfigurationUpgrader.py +++ b/src/lbaf/IO/lbsConfigurationUpgrader.py @@ -31,11 +31,13 @@ def bool_representer(dumper, value): text = "False" return dumper.represent_scalar('tag:yaml.org,2002:bool', text) + class UpgradeAction(Enum): """Upgrade action""" ADD_KEY = "add" REMOVE_KEY = "remove" + FORMAT_ONLY = "format_only" class ConfigurationDumper(yaml.Dumper): """Custom dumper to add indent before list items hyphens.""" @@ -43,16 +45,17 @@ class ConfigurationDumper(yaml.Dumper): def increase_indent(self, flow=False, indentless=False): return super(ConfigurationDumper, self).increase_indent(flow, False) + +ConfigurationDumper.add_representer(bool, bool_representer) + + class ConfigurationUpgrader: """This class enables to bulk upgrade configuration files by adding or removing keys.""" - __dumper: ConfigurationDumper __logger: Logger __sections: dict def __init__(self, logger: Logger): - self.__dumper = ConfigurationDumper - self.__dumper.add_representer(bool, bool_representer) self.__logger = logger self.__sections = cast(dict, ConfigurationValidator.allowed_keys(group=True)) @@ -68,7 +71,7 @@ def write_node(self, k: str, value: Any, yaml_file: IO, indent_size: int = 2): indent=indent_size, line_break='\n', sort_keys=False, - Dumper=self.__dumper + Dumper=ConfigurationDumper ).replace( '\n', '\n' + indent_str @@ -76,7 +79,16 @@ def write_node(self, k: str, value: Any, yaml_file: IO, indent_size: int = 2): if yaml_node.endswith('\n' + indent_str): yaml_node = yaml_node[:-(indent_size + 1)] else: - yaml_node = " " + yaml.representer.BaseRepresenter().represent_data(value).value + yaml_node = yaml.dump( + value, + indent=0, + Dumper=ConfigurationDumper, + explicit_start=None, + explicit_end=None + ) + if yaml_node.endswith('...\n'): + yaml_node = yaml_node[:-4] + yaml_node = ' ' + yaml_node.strip() yaml_file.write(yaml_node) yaml_file.write('\n') @@ -84,26 +96,27 @@ def upgrade( self, file_path: Path, action: UpgradeAction, - key: str, + key: str = None, value: str = None, value_type: str = "str" ) -> None: """Apply an upgrade to the given configuration file.""" self.__logger.debug("Upgrading file %s ...", file_path) key_path = None - if action == UpgradeAction.ADD_KEY: - self.__logger.debug("Add key `%s` with value `%s`", key, value) - elif action == UpgradeAction.REMOVE_KEY: - print(f"Remove key `{key}`") - key_path = key.split('.') - parsed_value = value - if value_type is not None: - type_v = locate(value_type) - if callable(type_v): - parsed_value = type_v(value) - - if key_path is None: - raise ValueError("The `key` must be a valid string") + if action != UpgradeAction.FORMAT_ONLY: + if action == UpgradeAction.ADD_KEY: + self.__logger.debug("Add key `%s` with value `%s`", key, value) + elif action == UpgradeAction.REMOVE_KEY: + print(f"Remove key `{key}`") + key_path = key.split('.') + parsed_value = value + if value_type is not None: + type_v = locate(value_type) + if callable(type_v): + parsed_value = type_v(value) + + if key_path is None and action != UpgradeAction.FORMAT_ONLY: + raise ValueError("The `key` must be a valid string") conf = None file_comment = None @@ -126,36 +139,36 @@ def upgrade( file_comment = file_comment.strip() # removes potential eol at the end yaml_file.seek(0) - yaml_content = yaml_file.read() conf = yaml.safe_load(yaml_content) node = conf - for i, key in enumerate(key_path): - is_leaf = i == len(key_path)-1 - # if node does not exist create it - if not key in node.keys(): - if action == UpgradeAction.ADD_KEY: - if is_leaf: + if action != UpgradeAction.FORMAT_ONLY: + for i, key in enumerate(key_path): + is_leaf = i == len(key_path)-1 + # if node does not exist create it + if not key in node.keys(): + if action == UpgradeAction.ADD_KEY: + if is_leaf: + node[key] = parsed_value + else: + new_branch = {} + new_node = new_branch + new_key_path = key_path[i+1:] + for j, new_key in enumerate(new_key_path): + is_leaf = j == len(new_key_path) - 1 + new_node[new_key] = parsed_value if is_leaf else {} + new_node = new_node[new_key] + node[key] = new_branch + elif is_leaf: + if action == UpgradeAction.REMOVE_KEY: + del node[key] + elif action == UpgradeAction.ADD_KEY: node[key] = parsed_value - else: - new_branch = {} - new_node = new_branch - new_key_path = key_path[i+1:] - for j, new_key in enumerate(new_key_path): - is_leaf = j == len(new_key_path) - 1 - new_node[new_key] = parsed_value if is_leaf else {} - new_node = new_node[new_key] - node[key] = new_branch - elif is_leaf: - if action == UpgradeAction.REMOVE_KEY: - del node[key] - else: - node[key] = parsed_value - # go next node in child tree - if is_leaf: - break + # go next node in child tree + if is_leaf: + break - node=node[key] + node=node[key] with open(file_path, 'w', encoding="utf-8") as yaml_file: if file_comment is not None: diff --git a/src/lbaf/IO/lbsConfigurationValidator.py b/src/lbaf/IO/lbsConfigurationValidator.py index ce4f4597a..fde0ea468 100644 --- a/src/lbaf/IO/lbsConfigurationValidator.py +++ b/src/lbaf/IO/lbsConfigurationValidator.py @@ -186,9 +186,9 @@ def allowed_keys(group: bool = False) -> Union[List[str], Dict[str, List[str]]] "work model": ["work_model"], "algorithm": ["brute_force_optimization", "algorithm"], "output": [ - "logging_level", "log_to_file", "overwrite_validator", "check_schema", "terminal_background", + "logging_level", "log_to_file", "overwrite_validator", "terminal_background", "generate_multimedia", "output_dir", "output_file_stem", - "LBAF_Viz" + "LBAF_Viz", "write_JSON" ] } diff --git a/src/lbaf/IO/lbsGridStreamer.py b/src/lbaf/IO/lbsGridStreamer.py index 54894d161..43c64a857 100644 --- a/src/lbaf/IO/lbsGridStreamer.py +++ b/src/lbaf/IO/lbsGridStreamer.py @@ -2,7 +2,7 @@ import vtk -from ..Utils.logging import get_logger, Logger +from ..Utils.logger import get_logger, Logger class GridStreamer: diff --git a/src/lbaf/Model/lbsPhase.py b/src/lbaf/Model/lbsPhase.py index aafd6b326..cafd6b005 100644 --- a/src/lbaf/Model/lbsPhase.py +++ b/src/lbaf/Model/lbsPhase.py @@ -11,7 +11,7 @@ from ..IO.lbsStatistics import print_subset_statistics, print_function_statistics, sampler from ..IO.lbsVTDataReader import LoadReader from ..Utils.exception_handler import exc_handler -from ..Utils.logging import get_logger +from ..Utils.logger import get_logger class Phase: diff --git a/src/lbaf/Model/lbsWorkModelBase.py b/src/lbaf/Model/lbsWorkModelBase.py index 44957f317..c389b0f0a 100644 --- a/src/lbaf/Model/lbsWorkModelBase.py +++ b/src/lbaf/Model/lbsWorkModelBase.py @@ -3,7 +3,7 @@ import sys from ..Utils.exception_handler import exc_handler -from ..Utils.logging import get_logger +from ..Utils.logger import get_logger class WorkModelBase: """An abstract base class of per-rank work model.""" diff --git a/src/lbaf/Utils/logging.py b/src/lbaf/Utils/logger.py similarity index 92% rename from src/lbaf/Utils/logging.py rename to src/lbaf/Utils/logger.py index 10767b091..7c180dd0a 100644 --- a/src/lbaf/Utils/logging.py +++ b/src/lbaf/Utils/logger.py @@ -1,9 +1,8 @@ """logging util module """ import logging -import logging.config import os -from logging import Formatter from typing import Union, List, Dict + from .colors import red, green, cyan, yellow @@ -13,18 +12,15 @@ "WARNING": logging.WARNING, "ERROR": logging.ERROR } - FORMAT_BASIC = "basic" """"%(levelname)s [%(module)s.%(funcName)s()] %(message)s""" FORMAT_EXTENDED = "extended" """"%(levelname)s [%(module)s.%(funcName)s()] [%(message)s]""" FORMATS = [ FORMAT_BASIC, FORMAT_EXTENDED ] - -# Logger type alias Logger = logging.Logger """Logger class""" -class CustomFormatter(Formatter): +class CustomFormatter(logging.Formatter): """Formatter able to write colored logs Colors are used to colorize log meta information such as: - the calling module name @@ -40,7 +36,7 @@ class CustomFormatter(Formatter): } _raw_formatter: None - _color_formatters: Dict[int,Formatter] = {} + _color_formatters: Dict[int,logging.Formatter] = {} _format:Union[FORMAT_BASIC, FORMAT_EXTENDED] _colored: bool = False @@ -56,7 +52,8 @@ def __init__(self, frmt: str, colored: bool = False): self._init_formatters() def _init_formatters(self): - """Initialize inner formatters for each supported logging level.""" + """Initialize inner formatters for each supported logging level""" + # 'basic' default format formats = { "prefix": "[%(module)s] ", "message": "%(message)s" } # 'extended' format @@ -66,12 +63,12 @@ def _init_formatters(self): # create inner formatter for each log level if self._colored: for level, colorizer in self.LOGGING_LEVEL_COLORS.items(): - self._color_formatters[level] = Formatter( + self._color_formatters[level] = logging.Formatter( colorizer(formats.get("prefix")) + formats.get("message") ) # or create raw formatter to use at any level else: - self._raw_formatter = Formatter(formats.get("prefix") + formats.get("message")) + self._raw_formatter = logging.Formatter(formats.get("prefix") + formats.get("message")) def format(self, record): formatter = None diff --git a/src/lbaf/Utils/vt_data_extractor.py b/src/lbaf/Utils/vt_data_extractor.py index 04c9e1955..e0e7eedba 100644 --- a/src/lbaf/Utils/vt_data_extractor.py +++ b/src/lbaf/Utils/vt_data_extractor.py @@ -1,15 +1,13 @@ -import os -import sys - +import json from multiprocessing.pool import Pool from multiprocessing import get_context - +import os +import sys import time -import json +from lbaf import PROJECT_PATH +from lbaf.Utils.exception_handler import exc_handler -from .exception_handler import exc_handler -from .. import PROJECT_PATH try: import brotli diff --git a/tests/test_configuration_validator.py b/tests/test_configuration_validator.py index 44c248759..04779bd7f 100644 --- a/tests/test_configuration_validator.py +++ b/tests/test_configuration_validator.py @@ -6,7 +6,7 @@ from lbaf.Utils.path import abspath from lbaf.IO.lbsConfigurationValidator import ConfigurationValidator -from lbaf.Utils.logging import get_logger +from lbaf.Utils.logger import get_logger class TestConfig(unittest.TestCase):