Merge pull request #893 from benmalef/add_logging_final_version

Add logging final version

.gitignore

@@ -36,3 +36,4 @@ tutorials/classification_medmnist_notebook/output_stats
 tutorials/classification_medmnist_notebook/model
 tutorials/classification_medmnist_notebook/dataset/*.csv
 testing/test_deploy
+tmp

GANDLF/entrypoints/anonymizer.py

@@ -12,6 +12,7 @@
 from GANDLF.anonymize import run_anonymizer
 from GANDLF.cli import copyrightMessage
 from GANDLF.entrypoints import append_copyright_to_help
+from GANDLF.utils.gandlf_logging import logger_setup
 
 
 def _anonymize_images(
@@ -77,6 +78,7 @@ def new_way(input_dir, config, modality, output_file):
     + "`gandlf_anonymizer` script would be deprecated soon."
 )
 def old_way():
+    logger_setup()
     parser = argparse.ArgumentParser(
         prog="GANDLF_Anonymize",
         formatter_class=argparse.RawTextHelpFormatter,

GANDLF/entrypoints/cli_tool.py

@@ -3,28 +3,18 @@
 import click
 from .subcommands import cli_subcommands
 from GANDLF.entrypoints import append_copyright_to_help
-
+from GANDLF.utils import logger_setup
 from GANDLF import version
 
 
-def setup_logging(loglevel):
-    logging.basicConfig(level=loglevel.upper())
-
-
 @click.group()
 @click.version_option(version, "--version", "-v", message="GANDLF Version: %(version)s")
-@click.option(
-    "--loglevel",
-    default="INFO",
-    help="Set the logging level (DEBUG, INFO, WARNING, ERROR, CRITICAL)",
-)
 @click.pass_context  # Pass the context to subcommands
 @append_copyright_to_help
-def gandlf(ctx, loglevel):
+def gandlf(ctx):
     """GANDLF command-line tool."""
     ctx.ensure_object(dict)
-    ctx.obj["LOGLEVEL"] = loglevel
-    setup_logging(loglevel)
+    logger_setup()
 
 
 # registers subcommands: `gandlf anonymizer`, `gandlf run`, etc.

GANDLF/entrypoints/collect_stats.py

@@ -14,6 +14,7 @@
 
 from GANDLF.cli import copyrightMessage
 from GANDLF.entrypoints import append_copyright_to_help
+from GANDLF.utils import logger_setup
 
 
 def plot_all(df_training, df_validation, df_testing, output_plot_dir):
@@ -205,6 +206,7 @@ def new_way(model_dir: str, output_dir: str):
     + "`gandlf_collectStats` script would be deprecated soon."
 )
 def old_way():
+    logger_setup()
     parser = argparse.ArgumentParser(
         prog="GANDLF_CollectStats",
         formatter_class=argparse.RawTextHelpFormatter,

GANDLF/entrypoints/config_generator.py

@@ -4,6 +4,7 @@
 
 from GANDLF.cli import config_generator, copyrightMessage
 from GANDLF.entrypoints import append_copyright_to_help
+from GANDLF.utils import logger_setup
 
 
 def _generate_config(config: str, strategy: str, output: str):
@@ -46,6 +47,7 @@ def new_way(config, strategy, output):
     + "`gandlf_configGenerator` script would be deprecated soon."
 )
 def old_way():
+    logger_setup()
     parser = argparse.ArgumentParser(
         prog="GANDLF_ConfigGenerator",
         formatter_class=argparse.RawTextHelpFormatter,

GANDLF/entrypoints/construct_csv.py

@@ -14,6 +14,7 @@
 from GANDLF.utils import writeTrainingCSV
 
 from GANDLF.cli import copyrightMessage
+from GANDLF.utils import logger_setup
 
 
 def _construct_csv(
@@ -119,6 +120,7 @@ def new_way(
     + "`gandlf_constructCSV` script would be deprecated soon."
 )
 def old_way():
+    logger_setup()
     parser = argparse.ArgumentParser(
         prog="GANDLF_ConstructCSV",
         formatter_class=argparse.RawTextHelpFormatter,

GANDLF/entrypoints/debug_info.py

@@ -7,6 +7,7 @@
 from GANDLF import __version__
 from GANDLF.entrypoints import append_copyright_to_help
 from GANDLF.utils import get_git_hash
+from GANDLF.utils import logger_setup
 
 
 def _debug_info():
@@ -38,6 +39,7 @@ def new_way():
 )
 def old_way():
     _debug_info()
+    logger_setup()
 
 
 if __name__ == "__main__":

GANDLF/entrypoints/deploy.py

@@ -16,6 +16,7 @@
     copyrightMessage,
 )
 from GANDLF.entrypoints import append_copyright_to_help
+from GANDLF.utils import logger_setup
 
 
 def _deploy(
@@ -157,6 +158,7 @@ def new_way(
     + "`gandlf_deploy` script would be deprecated soon."
 )
 def old_way():
+    logger_setup()
     parser = argparse.ArgumentParser(
         prog="GANDLF_Deploy",
         formatter_class=argparse.RawTextHelpFormatter,

GANDLF/entrypoints/generate_metrics.py

@@ -10,6 +10,7 @@
 from GANDLF.cli import copyrightMessage
 from GANDLF.cli.generate_metrics import generate_metrics_dict
 from GANDLF.entrypoints import append_copyright_to_help
+from GANDLF.utils import logger_setup
 
 
 def _generate_metrics(
@@ -81,6 +82,7 @@ def new_way(
     + "`gandlf_generateMetrics` script would be deprecated soon."
 )
 def old_way():
+    logger_setup()
     parser = argparse.ArgumentParser(
         prog="GANDLF_Metrics",
         formatter_class=argparse.RawTextHelpFormatter,

GANDLF/entrypoints/optimize_model.py

@@ -9,6 +9,7 @@
 
 from GANDLF.cli import copyrightMessage, post_training_model_optimization
 from GANDLF.entrypoints import append_copyright_to_help
+from GANDLF.utils import logger_setup
 
 
 def _optimize_model(
@@ -60,6 +61,7 @@ def new_way(
     + "`gandlf_optimizeModel` script would be deprecated soon."
 )
 def old_way():
+    logger_setup()
     parser = argparse.ArgumentParser(
         prog="GANDLF_OptimizeModel",
         formatter_class=argparse.RawTextHelpFormatter,

GANDLF/entrypoints/patch_miner.py

@@ -10,6 +10,7 @@
 from GANDLF.cli.patch_extraction import patch_extraction
 from GANDLF.cli import copyrightMessage
 from GANDLF.entrypoints import append_copyright_to_help
+from GANDLF.utils import logger_setup
 
 
 def _mine_patches(input_path: str, output_dir: str, config: Optional[str]):
@@ -55,6 +56,7 @@ def new_way(input_csv: str, output_dir: str, config: Optional[str]):
     + "`gandlf_patchMiner` script would be deprecated soon."
 )
 def old_way():
+    logger_setup()
     parser = argparse.ArgumentParser(
         prog="GANDLF_PatchMiner",
         formatter_class=argparse.RawTextHelpFormatter,

GANDLF/entrypoints/preprocess.py

@@ -8,6 +8,7 @@
 from deprecated import deprecated
 from GANDLF.cli import preprocess_and_save, copyrightMessage
 from GANDLF.entrypoints import append_copyright_to_help
+from GANDLF.utils import logger_setup
 
 
 def _preprocess(
@@ -110,6 +111,7 @@ def new_way(
     + "`gandlf_preprocess` script would be deprecated soon."
 )
 def old_way():
+    logger_setup()
     parser = argparse.ArgumentParser(
         prog="GANDLF_Preprocess",
         formatter_class=argparse.RawTextHelpFormatter,

GANDLF/entrypoints/recover_config.py

@@ -9,6 +9,7 @@
 
 from GANDLF.cli import copyrightMessage, recover_config
 from GANDLF.entrypoints import append_copyright_to_help
+from GANDLF.utils import logger_setup
 
 
 def _recover_config(model_dir: Optional[str], mlcube: bool, output_file: str):
@@ -63,6 +64,7 @@ def new_way(model_dir, mlcube, output_file):
     + "`gandlf_recoverConfig` script would be deprecated soon."
 )
 def old_way():
+    logger_setup()
     parser = argparse.ArgumentParser(
         prog="GANDLF_RecoverConfig",
         formatter_class=argparse.RawTextHelpFormatter,

GANDLF/entrypoints/run.py

@@ -14,6 +14,7 @@
 from GANDLF import version
 from GANDLF.cli import main_run, copyrightMessage
 from GANDLF.entrypoints import append_copyright_to_help
+from GANDLF.utils import logger_setup
 
 
 def _run(
@@ -184,6 +185,7 @@ def new_way(
     + "`gandlf_run` script would be deprecated soon."
 )
 def old_way():
+    logger_setup()
     parser = argparse.ArgumentParser(
         prog="GANDLF",
         formatter_class=argparse.RawTextHelpFormatter,

GANDLF/entrypoints/split_csv.py

@@ -12,6 +12,7 @@
 
 from GANDLF.cli import copyrightMessage, split_data_and_save_csvs
 from GANDLF.entrypoints import append_copyright_to_help
+from GANDLF.utils import logger_setup
 
 
 def _split_csv(input_csv: str, output_dir: str, config_path: Optional[str]):
@@ -64,6 +65,7 @@ def new_way(input_csv: str, output_dir: str, config: Optional[str]):
     + "`gandlf_splitCSV` script would be deprecated soon."
 )
 def old_way():
+    logger_setup()
     parser = argparse.ArgumentParser(
         prog="GANDLF_SplitCSV",
         formatter_class=argparse.RawTextHelpFormatter,

GANDLF/entrypoints/verify_install.py

@@ -6,6 +6,7 @@
 from deprecated import deprecated
 
 from GANDLF.entrypoints import append_copyright_to_help
+from GANDLF.utils import logger_setup
 
 
 def _verify_install():
@@ -42,6 +43,7 @@ def new_way():
     + "`gandlf_verifyInstall` script would be deprecated soon."
 )
 def old_way():
+    logger_setup()
     argparse.ArgumentParser(
         prog="GANDLF_VerifyInstall",
         formatter_class=argparse.RawTextHelpFormatter,

GANDLF/logging_config.yaml

@@ -0,0 +1,47 @@
+version: 1
+formatters:
+  detailed:
+    format: "%(asctime)s - %(name)s - %(levelname)s - %(module)s:%(funcName)s:%(lineno)d - %(message)s"
+    datefmt: "%Y-%m-%d %H:%M:%S"
+    log_colors:
+      DEBUG: "white"
+      INFO: "green"
+      WARNING: "yellow"
+      ERROR: "red"
+      CRITICAL: "bold_red"
+  simple:
+    (): colorlog.ColoredFormatter
+    format: "%(log_color)s%(asctime)s - %(levelname)s - %(message)s"
+    datefmt: "%Y-%m-%d %H:%M:%S"
+filters:
+  warnings_filter:
+    (): logging.Filter
+    name: "py.warnings"
+  info_only_filter:
+    (): GANDLF.utils.gandlf_logging.InfoOnlyFilter
+handlers:
+  stdoutHandler: # only display info level
+    class: logging.StreamHandler
+    level: INFO
+    formatter: simple
+    filters: [info_only_filter]
+    stream: ext://sys.stdout
+  stderrHandler: # display warning and above messages
+    class: logging.StreamHandler
+    level: WARNING
+    formatter: detailed
+    stream: ext://sys.stderr
+  rotatingFileHandler:
+    class: logging.handlers.RotatingFileHandler
+    level: DEBUG
+    formatter: detailed
+    maxBytes: 10485760
+    backupCount: 2
+loggers: # you can add your customized logger
+  debug_logger:
+    level: DEBUG
+    handlers: [stdoutHandler, rotatingFileHandler, stderrHandler]
+    propagate: no
+root:
+  level: DEBUG
+  handlers: [stdoutHandler, rotatingFileHandler, stderrHandler]

GANDLF/utils/__init__.py

@@ -68,3 +68,4 @@
 )
 
 from .data_splitter import split_data
+from .gandlf_logging import logger_setup, InfoOnlyFilter

GANDLF/utils/gandlf_logging.py

@@ -0,0 +1,61 @@
+import logging
+import yaml
+from pathlib import Path
+from importlib import resources
+import tempfile
+from GANDLF.utils import get_unique_timestamp
+
+
+def _create_tmp_log_file():
+    tmp_dir = Path(tempfile.gettempdir())
+    log_dir = Path.joinpath(tmp_dir, ".gandlf")
+    log_dir.mkdir(parents=True, exist_ok=True)
+    log_file = Path.joinpath(log_dir, get_unique_timestamp() + ".log")
+    return log_file
+
+
+def _create_log_file(log_file):
+    log_file = Path(log_file)
+    log_file.write_text("Starting GaNDLF logging session \n")
+
+
+def _configure_logging_with_logfile(log_file, config_path):
+    with resources.open_text("GANDLF", config_path) as file:
+        config_dict = yaml.safe_load(file)
+        config_dict["handlers"]["rotatingFileHandler"]["filename"] = str(log_file)
+        logging.config.dictConfig(config_dict)
+
+
+def logger_setup(log_file=None, config_path="logging_config.yaml") -> None:
+    """
+    It sets up the logger. Reads from logging_config.
+
+    Args:
+        log_file (str): dir path for saving the logs, defaults to `None`, at which time logs are flushed to console.
+        config_path (str): file path for the configuration
+
+    """
+
+    logging.captureWarnings(True)
+    log_tmp_file = log_file
+    if log_file is None:  # create tmp file
+        log_tmp_file = _create_tmp_log_file()
+        logging.info(f"The logs are saved in {log_tmp_file}")
+    _create_log_file(log_tmp_file)
+    _configure_logging_with_logfile(log_tmp_file, config_path)
+
+
+class InfoOnlyFilter(logging.Filter):
+    """
+    Display only INFO messages.
+    """
+
+    def filter(self, record):
+        """
+        Determines if the specified record is to be logged.
+        Args:
+            record (logging.LogRecord): The log record to be evaluated.
+        Returns:
+            bool: True if the log record should be processed, False otherwise.
+        """
+        return record.levelno == logging.INFO

MANIFEST.in

@@ -4,3 +4,4 @@ exclude *.toml
 include setup.py
 include .dockerignore
 include Dockerfile-*
+include logging_config.yml