Add logging final version

.gitignore

@@ -36,3 +36,4 @@ tutorials/classification_medmnist_notebook/output_stats
 tutorials/classification_medmnist_notebook/model
 tutorials/classification_medmnist_notebook/dataset/*.csv
 testing/test_deploy
+tmp

GANDLF/compute/forward_pass.py

@@ -113,7 +113,7 @@ def validate_network(
         tqdm(valid_dataloader, desc="Looping over " + mode + " data")
     ):
         if params["verbose"]:
-            print("== Current subject:", subject["subject_id"], flush=True)
+            logging.debug(f'== Current subject: {subject["subject_id"]}')
 
         # ensure spacing is always present in params and is always subject-specific
         params["subject_spacing"] = None

GANDLF/entrypoints/anonymizer.py

@@ -12,6 +12,7 @@
 from GANDLF.anonymize import run_anonymizer
 from GANDLF.cli import copyrightMessage
 from GANDLF.entrypoints import append_copyright_to_help
+from GANDLF.utils.gandlf_logger import gandlf_logger_setup
 
 
 def _anonymize_images(
@@ -77,6 +78,7 @@ def new_way(input_dir, config, modality, output_file):
     + "`gandlf_anonymizer` script would be deprecated soon."
 )
 def old_way():
+    gandlf_logger_setup()
     parser = argparse.ArgumentParser(
         prog="GANDLF_Anonymize",
         formatter_class=argparse.RawTextHelpFormatter,

GANDLF/entrypoints/cli_tool.py

@@ -3,7 +3,7 @@
 import click
 from .subcommands import cli_subcommands
 from GANDLF.entrypoints import append_copyright_to_help
-
+from GANDLF.utils import gandlf_logger_setup
 from GANDLF import version
 
 
@@ -24,7 +24,8 @@ def gandlf(ctx, loglevel):
     """GANDLF command-line tool."""
     ctx.ensure_object(dict)
     ctx.obj["LOGLEVEL"] = loglevel
-    setup_logging(loglevel)
+    # setup_logging(loglevel)
+    gandlf_logger_setup()
 
 
 # registers subcommands: `gandlf anonymizer`, `gandlf run`, etc.

GANDLF/entrypoints/collect_stats.py

@@ -14,6 +14,7 @@
 
 from GANDLF.cli import copyrightMessage
 from GANDLF.entrypoints import append_copyright_to_help
+from GANDLF.utils import gandlf_logger_setup
 
 
 def plot_all(df_training, df_validation, df_testing, output_plot_dir):
@@ -205,6 +206,7 @@ def new_way(model_dir: str, output_dir: str):
     + "`gandlf_collectStats` script would be deprecated soon."
 )
 def old_way():
+    gandlf_logger_setup()
     parser = argparse.ArgumentParser(
         prog="GANDLF_CollectStats",
         formatter_class=argparse.RawTextHelpFormatter,

GANDLF/entrypoints/config_generator.py

@@ -4,6 +4,7 @@
 
 from GANDLF.cli import config_generator, copyrightMessage
 from GANDLF.entrypoints import append_copyright_to_help
+from GANDLF.utils.gandlf_logger import gandlf_logger_setup
 
 
 def _generate_config(config: str, strategy: str, output: str):
@@ -46,6 +47,7 @@ def new_way(config, strategy, output):
     + "`gandlf_configGenerator` script would be deprecated soon."
 )
 def old_way():
+    gandlf_logger_setup()
     parser = argparse.ArgumentParser(
         prog="GANDLF_ConfigGenerator",
         formatter_class=argparse.RawTextHelpFormatter,

GANDLF/entrypoints/construct_csv.py

@@ -14,6 +14,7 @@
 from GANDLF.utils import writeTrainingCSV
 
 from GANDLF.cli import copyrightMessage
+from GANDLF.utils.gandlf_logger import gandlf_logger_setup
 
 
 def _construct_csv(
@@ -119,6 +120,7 @@ def new_way(
     + "`gandlf_constructCSV` script would be deprecated soon."
 )
 def old_way():
+    gandlf_logger_setup()
     parser = argparse.ArgumentParser(
         prog="GANDLF_ConstructCSV",
         formatter_class=argparse.RawTextHelpFormatter,

GANDLF/entrypoints/debug_info.py

@@ -7,6 +7,7 @@
 from GANDLF import __version__
 from GANDLF.entrypoints import append_copyright_to_help
 from GANDLF.utils import get_git_hash
+from GANDLF.utils import gandlf_logger_setup
 
 
 def _debug_info():
@@ -38,6 +39,7 @@ def new_way():
 )
 def old_way():
     _debug_info()
+    gandlf_logger_setup()
 
 
 if __name__ == "__main__":

GANDLF/entrypoints/deploy.py

@@ -16,6 +16,7 @@
     copyrightMessage,
 )
 from GANDLF.entrypoints import append_copyright_to_help
+from GANDLF.utils.gandlf_logger import gandlf_logger_setup
 
 
 def _deploy(
@@ -157,6 +158,7 @@ def new_way(
     + "`gandlf_deploy` script would be deprecated soon."
 )
 def old_way():
+    gandlf_logger_setup()
     parser = argparse.ArgumentParser(
         prog="GANDLF_Deploy",
         formatter_class=argparse.RawTextHelpFormatter,

GANDLF/entrypoints/generate_metrics.py

@@ -11,6 +11,7 @@
 from GANDLF.cli import copyrightMessage
 from GANDLF.cli.generate_metrics import generate_metrics_dict
 from GANDLF.entrypoints import append_copyright_to_help
+from GANDLF.utils.gandlf_logger import gandlf_logger_setup
 
 
 def _generate_metrics(
@@ -82,6 +83,7 @@ def new_way(
     + "`gandlf_generateMetrics` script would be deprecated soon."
 )
 def old_way():
+    gandlf_logger_setup()
     parser = argparse.ArgumentParser(
         prog="GANDLF_Metrics",
         formatter_class=argparse.RawTextHelpFormatter,

GANDLF/entrypoints/optimize_model.py

@@ -9,6 +9,7 @@
 
 from GANDLF.cli import copyrightMessage, post_training_model_optimization
 from GANDLF.entrypoints import append_copyright_to_help
+from GANDLF.utils.gandlf_logger import gandlf_logger_setup
 
 
 def _optimize_model(
@@ -60,6 +61,7 @@ def new_way(
     + "`gandlf_optimizeModel` script would be deprecated soon."
 )
 def old_way():
+    gandlf_logger_setup()
     parser = argparse.ArgumentParser(
         prog="GANDLF_OptimizeModel",
         formatter_class=argparse.RawTextHelpFormatter,

GANDLF/entrypoints/patch_miner.py

@@ -10,6 +10,7 @@
 from GANDLF.cli.patch_extraction import patch_extraction
 from GANDLF.cli import copyrightMessage
 from GANDLF.entrypoints import append_copyright_to_help
+from GANDLF.utils.gandlf_logger import gandlf_logger_setup
 
 
 def _mine_patches(input_path: str, output_dir: str, config: Optional[str]):
@@ -55,6 +56,7 @@ def new_way(input_csv: str, output_dir: str, config: Optional[str]):
     + "`gandlf_patchMiner` script would be deprecated soon."
 )
 def old_way():
+    gandlf_logger_setup()
     parser = argparse.ArgumentParser(
         prog="GANDLF_PatchMiner",
         formatter_class=argparse.RawTextHelpFormatter,

GANDLF/entrypoints/preprocess.py

@@ -8,6 +8,7 @@
 from deprecated import deprecated
 from GANDLF.cli import preprocess_and_save, copyrightMessage
 from GANDLF.entrypoints import append_copyright_to_help
+from GANDLF.utils.gandlf_logger import gandlf_logger_setup
 
 
 def _preprocess(
@@ -110,6 +111,7 @@ def new_way(
     + "`gandlf_preprocess` script would be deprecated soon."
 )
 def old_way():
+    gandlf_logger_setup()
     parser = argparse.ArgumentParser(
         prog="GANDLF_Preprocess",
         formatter_class=argparse.RawTextHelpFormatter,

GANDLF/entrypoints/recover_config.py

@@ -9,6 +9,7 @@
 
 from GANDLF.cli import copyrightMessage, recover_config
 from GANDLF.entrypoints import append_copyright_to_help
+from GANDLF.utils.gandlf_logger import gandlf_logger_setup
 
 
 def _recover_config(model_dir: Optional[str], mlcube: bool, output_file: str):
@@ -63,6 +64,7 @@ def new_way(model_dir, mlcube, output_file):
     + "`gandlf_recoverConfig` script would be deprecated soon."
 )
 def old_way():
+    gandlf_logger_setup()
     parser = argparse.ArgumentParser(
         prog="GANDLF_RecoverConfig",
         formatter_class=argparse.RawTextHelpFormatter,

GANDLF/entrypoints/run.py

@@ -14,6 +14,7 @@
 from GANDLF import version
 from GANDLF.cli import main_run, copyrightMessage
 from GANDLF.entrypoints import append_copyright_to_help
+from GANDLF.utils.gandlf_logger import gandlf_logger_setup
 
 
 def _run(
@@ -184,6 +185,7 @@ def new_way(
     + "`gandlf_run` script would be deprecated soon."
 )
 def old_way():
+    gandlf_logger_setup()
     parser = argparse.ArgumentParser(
         prog="GANDLF",
         formatter_class=argparse.RawTextHelpFormatter,

GANDLF/entrypoints/split_csv.py

@@ -12,6 +12,7 @@
 
 from GANDLF.cli import copyrightMessage, split_data_and_save_csvs
 from GANDLF.entrypoints import append_copyright_to_help
+from GANDLF.utils.gandlf_logger import gandlf_logger_setup
 
 
 def _split_csv(input_csv: str, output_dir: str, config_path: Optional[str]):
@@ -64,6 +65,7 @@ def new_way(input_csv: str, output_dir: str, config: Optional[str]):
     + "`gandlf_splitCSV` script would be deprecated soon."
 )
 def old_way():
+    gandlf_logger_setup()
     parser = argparse.ArgumentParser(
         prog="GANDLF_SplitCSV",
         formatter_class=argparse.RawTextHelpFormatter,

GANDLF/entrypoints/verify_install.py

@@ -6,6 +6,7 @@
 from deprecated import deprecated
 
 from GANDLF.entrypoints import append_copyright_to_help
+from GANDLF.utils.gandlf_logger import gandlf_logger_setup
 
 
 def _verify_install():
@@ -42,6 +43,7 @@ def new_way():
     + "`gandlf_verifyInstall` script would be deprecated soon."
 )
 def old_way():
+    gandlf_logger_setup()
     argparse.ArgumentParser(
         prog="GANDLF_VerifyInstall",
         formatter_class=argparse.RawTextHelpFormatter,

GANDLF/logging_config.yaml

@@ -0,0 +1,47 @@
+version: 1
+formatters:
+  detailed:
+    format: "%(asctime)s - %(name)s - %(levelname)s - %(module)s:%(funcName)s:%(lineno)d - %(message)s"
+    datefmt: "%Y-%m-%d %H:%M:%S"
+    log_colors:
+      DEBUG: "white"
+      INFO: "green"
+      WARNING: "yellow"
+      ERROR: "red"
+      CRITICAL: "bold_red"
+  simple:
+    (): colorlog.ColoredFormatter
+    format: "%(log_color)s%(asctime)s - %(levelname)s - %(message)s"
+    datefmt: "%Y-%m-%d %H:%M:%S"
+filters:
+  warnings_filter:
+    (): logging.Filter
+    name: "py.warnings"
+  info_only_filter:
+    (): GANDLF.utils.gandlf_logger.InfoOnlyFilter
+handlers:
+  stdoutHandler: # only display info level
+    class: logging.StreamHandler
+    level: INFO
+    formatter: simple
+    filters: [info_only_filter]
+    stream: ext://sys.stdout
+  stderrHandler: # display warning and above messages
+    class: logging.StreamHandler
+    level: WARNING
+    formatter: detailed
+    stream: ext://sys.stderr
+  rotatingFileHandler:
+    class: logging.handlers.RotatingFileHandler
+    level: DEBUG
+    formatter: detailed
+    maxBytes: 10485760
+    backupCount: 2
+loggers: # you can add your customized logger
+  debug_logger:
+    level: DEBUG
+    handlers: [stdoutHandler, rotatingFileHandler, stderrHandler]
+    propagate: no
+root:
+  level: DEBUG
+  handlers: [stdoutHandler, rotatingFileHandler, stderrHandler]

GANDLF/utils/__init__.py

@@ -68,3 +68,4 @@
 )
 
 from .data_splitter import split_data
+from .gandlf_logger import gandlf_logger_setup, InfoOnlyFilter

GANDLF/utils/gandlf_logger.py

@@ -0,0 +1,87 @@
+import logging
+import yaml
+from pathlib import Path
+from importlib import resources
+import colorlog
+import tempfile
+from GANDLF.utils import get_unique_timestamp
+
+
+def _flush_to_console():
+    formatter = colorlog.ColoredFormatter(
+        "%(log_color)s%(asctime)s - %(levelname)s - %(message)s",
+        datefmt="%Y-%m-%d %H:%M:%S",
+        log_colors={
+            "DEBUG": "blue",
+            "INFO": "green",
+            "WARNING": "yellow",
+            "ERROR": "red",
+            "CRITICAL": "bold_red",
+        },
+    )
+    console_handler = logging.StreamHandler()
+    console_handler.setFormatter(formatter)
+    logging.root.setLevel(logging.DEBUG)
+    logging.root.addHandler(console_handler)
+
+
+def _create_tmp_log_file():
+    tmp_dir = Path(tempfile.gettempdir())
+    log_dir = Path.joinpath(tmp_dir, ".gandlf")
+    log_dir.mkdir(parents=True, exist_ok=True)
+    log_file = Path.joinpath(log_dir, get_unique_timestamp() + ".log")
+    _create_log_file(log_file)
+    return log_file
+
+
+def _create_log_file(log_file):
+    log_file = Path(log_file)
+    log_file.write_text("Starting GaNDLF logging session \n")
+
+
+def _save_logs_in_file(log_file, config_path):
+    with resources.open_text("GANDLF", config_path) as file:
+        config_dict = yaml.safe_load(file)
+        config_dict["handlers"]["rotatingFileHandler"]["filename"] = str(log_file)
+        logging.config.dictConfig(config_dict)
+
+
+def gandlf_logger_setup(log_file=None, config_path="logging_config.yaml"):
+    """
+    It sets up the logger. Reads from logging_config.
+
+    Args:
+        log_file (str): dir path for saving the logs, defaults to `None`, at which time logs are flushed to console.
+        config_path (str): file path for the configuration
+
+    """
+
+    logging.captureWarnings(True)
+    try:
+        if log_file is None:  # create tmp file
+            log_tmp_file = _create_tmp_log_file()
+            _save_logs_in_file(log_tmp_file, config_path)
+            logging.info(f"The logs are saved in {log_tmp_file}")
+        else:  # create the log file
+            _create_log_file(log_file)
+            _save_logs_in_file(log_file, config_path)
+    except Exception as e:
+        _flush_to_console()
+        logging.error(f"log_file:{e}")
+        logging.warning("The logs will be flushed to console")
+
+
+class InfoOnlyFilter(logging.Filter):
+    """
+    Display only INFO messages.
+    """
+
+    def filter(self, record):
+        """
+        Determines if the specified record is to be logged.
+        Args:
+            record (logging.LogRecord): The log record to be evaluated.
+        Returns:
+            bool: True if the log record should be processed, False otherwise.
+        """
+        return record.levelno == logging.INFO