From 684613e0df3963e4541583245bebfa0aafc13271 Mon Sep 17 00:00:00 2001 From: Robbe Sneyders Date: Fri, 23 Jun 2023 09:22:30 +0200 Subject: [PATCH] Centralize logging configuration (#229) I was getting double logs with the local runner and was never a fan of the `configure_logging` utility function which had to be called by the user. This sets the logging format and level centrally in a way that doesn't overwrite already configured handlers and can easily be overwritten for each separate logger by the user. --- components/caption_images/src/main.py | 2 -- components/download_images/src/main.py | 2 -- .../src/main.py | 2 -- components/filter_comments/src/main.py | 2 -- components/filter_line_length/src/main.py | 2 -- components/image_cropping/src/main.py | 2 -- components/image_embedding/src/main.py | 2 -- .../image_resolution_extraction/src/main.py | 2 -- .../image_resolution_filtering/src/main.py | 2 -- components/load_from_hf_hub/src/main.py | 2 -- components/pii_redaction/src/main.py | 2 -- .../prompt_based_laion_retrieval/src/main.py | 2 -- components/segment_images/src/main.py | 2 -- components/write_to_hf_hub/src/main.py | 2 -- .../components/generate_prompts/src/main.py | 2 -- .../controlnet-interior-design/pipeline.py | 2 -- .../filter_text_complexity/src/main.py | 2 -- examples/pipelines/datacomp/pipeline.py | 2 -- .../finetune_stable_diffusion/pipeline.py | 2 -- examples/pipelines/starcoder/pipeline.py | 2 -- fondant/__init__.py | 5 ++++ fondant/logger.py | 23 ----------------- tests/test_logger.py | 25 ------------------- 23 files changed, 5 insertions(+), 88 deletions(-) delete mode 100644 fondant/logger.py delete mode 100644 tests/test_logger.py diff --git a/components/caption_images/src/main.py b/components/caption_images/src/main.py index cd73f2ea9..ff9cec607 100644 --- a/components/caption_images/src/main.py +++ b/components/caption_images/src/main.py @@ -10,9 +10,7 @@ from transformers import BatchEncoding, BlipForConditionalGeneration, BlipProcessor from fondant.component import PandasTransformComponent -from fondant.logger import configure_logging -configure_logging() logger = logging.getLogger(__name__) diff --git a/components/download_images/src/main.py b/components/download_images/src/main.py index 54d529aca..307ca052b 100644 --- a/components/download_images/src/main.py +++ b/components/download_images/src/main.py @@ -14,9 +14,7 @@ from resizer import Resizer from fondant.component import PandasTransformComponent -from fondant.logger import configure_logging -configure_logging() logger = logging.getLogger(__name__) diff --git a/components/embedding_based_laion_retrieval/src/main.py b/components/embedding_based_laion_retrieval/src/main.py index ae2c4f899..54e21831d 100644 --- a/components/embedding_based_laion_retrieval/src/main.py +++ b/components/embedding_based_laion_retrieval/src/main.py @@ -9,9 +9,7 @@ from clip_client import ClipClient, Modality from fondant.component import PandasTransformComponent -from fondant.logger import configure_logging -configure_logging() logger = logging.getLogger(__name__) diff --git a/components/filter_comments/src/main.py b/components/filter_comments/src/main.py index 31b5c58f1..3a5282866 100644 --- a/components/filter_comments/src/main.py +++ b/components/filter_comments/src/main.py @@ -8,9 +8,7 @@ from utils.text_extraction import get_comments_to_code_ratio from fondant.component import DaskTransformComponent -from fondant.logger import configure_logging -configure_logging() logger = logging.getLogger(__name__) diff --git a/components/filter_line_length/src/main.py b/components/filter_line_length/src/main.py index 98bf87f14..ed04b5564 100644 --- a/components/filter_line_length/src/main.py +++ b/components/filter_line_length/src/main.py @@ -4,9 +4,7 @@ import dask.dataframe as dd from fondant.component import DaskTransformComponent -from fondant.logger import configure_logging -configure_logging() logger = logging.getLogger(__name__) diff --git a/components/image_cropping/src/main.py b/components/image_cropping/src/main.py index aebe43cd8..664d6f19d 100644 --- a/components/image_cropping/src/main.py +++ b/components/image_cropping/src/main.py @@ -9,9 +9,7 @@ from PIL import Image from fondant.component import DaskTransformComponent -from fondant.logger import configure_logging -configure_logging() logger = logging.getLogger(__name__) diff --git a/components/image_embedding/src/main.py b/components/image_embedding/src/main.py index 66c5c1e2f..7fcc0d796 100644 --- a/components/image_embedding/src/main.py +++ b/components/image_embedding/src/main.py @@ -10,9 +10,7 @@ from transformers import CLIPProcessor, CLIPVisionModelWithProjection from fondant.component import PandasTransformComponent -from fondant.logger import configure_logging -configure_logging() logger = logging.getLogger(__name__) diff --git a/components/image_resolution_extraction/src/main.py b/components/image_resolution_extraction/src/main.py index 97d29306a..8b0f03314 100644 --- a/components/image_resolution_extraction/src/main.py +++ b/components/image_resolution_extraction/src/main.py @@ -8,9 +8,7 @@ import pandas as pd from fondant.component import PandasTransformComponent -from fondant.logger import configure_logging -configure_logging() logger = logging.getLogger(__name__) diff --git a/components/image_resolution_filtering/src/main.py b/components/image_resolution_filtering/src/main.py index 44082ae0d..63b612b6c 100644 --- a/components/image_resolution_filtering/src/main.py +++ b/components/image_resolution_filtering/src/main.py @@ -4,9 +4,7 @@ import dask.dataframe as dd from fondant.component import DaskTransformComponent -from fondant.logger import configure_logging -configure_logging() logger = logging.getLogger(__name__) diff --git a/components/load_from_hf_hub/src/main.py b/components/load_from_hf_hub/src/main.py index fd21aae22..0f3d774db 100644 --- a/components/load_from_hf_hub/src/main.py +++ b/components/load_from_hf_hub/src/main.py @@ -5,9 +5,7 @@ import dask.dataframe as dd from fondant.component import LoadComponent -from fondant.logger import configure_logging -configure_logging() logger = logging.getLogger(__name__) diff --git a/components/pii_redaction/src/main.py b/components/pii_redaction/src/main.py index 9c28200f2..7404f7626 100644 --- a/components/pii_redaction/src/main.py +++ b/components/pii_redaction/src/main.py @@ -8,9 +8,7 @@ from pii_redaction import redact_pii from fondant.component import DaskTransformComponent -from fondant.logger import configure_logging -configure_logging() logger = logging.getLogger(__name__) diff --git a/components/prompt_based_laion_retrieval/src/main.py b/components/prompt_based_laion_retrieval/src/main.py index 44e325424..d1ac482f2 100644 --- a/components/prompt_based_laion_retrieval/src/main.py +++ b/components/prompt_based_laion_retrieval/src/main.py @@ -8,9 +8,7 @@ from clip_client import ClipClient, Modality from fondant.component import PandasTransformComponent -from fondant.logger import configure_logging -configure_logging() logger = logging.getLogger(__name__) diff --git a/components/segment_images/src/main.py b/components/segment_images/src/main.py index c28a4bb1e..a509e8249 100644 --- a/components/segment_images/src/main.py +++ b/components/segment_images/src/main.py @@ -11,9 +11,7 @@ from transformers import AutoModelForSemanticSegmentation, BatchFeature, SegformerImageProcessor from fondant.component import PandasTransformComponent -from fondant.logger import configure_logging -configure_logging() logger = logging.getLogger(__name__) diff --git a/components/write_to_hf_hub/src/main.py b/components/write_to_hf_hub/src/main.py index cd983a7af..6592255e1 100644 --- a/components/write_to_hf_hub/src/main.py +++ b/components/write_to_hf_hub/src/main.py @@ -11,9 +11,7 @@ from PIL import Image from fondant.component import WriteComponent -from fondant.logger import configure_logging -configure_logging() logger = logging.getLogger(__name__) diff --git a/examples/pipelines/controlnet-interior-design/components/generate_prompts/src/main.py b/examples/pipelines/controlnet-interior-design/components/generate_prompts/src/main.py index e10c749e6..7b0369098 100644 --- a/examples/pipelines/controlnet-interior-design/components/generate_prompts/src/main.py +++ b/examples/pipelines/controlnet-interior-design/components/generate_prompts/src/main.py @@ -9,9 +9,7 @@ import pandas as pd from fondant.component import LoadComponent -from fondant.logger import configure_logging -configure_logging() logger = logging.getLogger(__name__) interior_styles = [ diff --git a/examples/pipelines/controlnet-interior-design/pipeline.py b/examples/pipelines/controlnet-interior-design/pipeline.py index 355a1ffc6..561d25e83 100644 --- a/examples/pipelines/controlnet-interior-design/pipeline.py +++ b/examples/pipelines/controlnet-interior-design/pipeline.py @@ -13,9 +13,7 @@ Pipeline, Client, ) -from fondant.logger import configure_logging -configure_logging() logger = logging.getLogger(__name__) # General configs pipeline_name = "controlnet-pipeline" diff --git a/examples/pipelines/datacomp/components/filter_text_complexity/src/main.py b/examples/pipelines/datacomp/components/filter_text_complexity/src/main.py index ec997adf3..865597d0b 100644 --- a/examples/pipelines/datacomp/components/filter_text_complexity/src/main.py +++ b/examples/pipelines/datacomp/components/filter_text_complexity/src/main.py @@ -12,9 +12,7 @@ from spacy.symbols import nsubj, VERB from fondant.component import PandasTransformComponent -from fondant.logger import configure_logging -configure_logging() logger = logging.getLogger(__name__) diff --git a/examples/pipelines/datacomp/pipeline.py b/examples/pipelines/datacomp/pipeline.py index fbec529b4..71f0a47ba 100644 --- a/examples/pipelines/datacomp/pipeline.py +++ b/examples/pipelines/datacomp/pipeline.py @@ -9,9 +9,7 @@ from fondant.compiler import DockerCompiler from fondant.pipeline import ComponentOp, Pipeline, Client -from fondant.logger import configure_logging -configure_logging() logger = logging.getLogger(__name__) # Initialize pipeline and client diff --git a/examples/pipelines/finetune_stable_diffusion/pipeline.py b/examples/pipelines/finetune_stable_diffusion/pipeline.py index 5751583de..5ad9f0a70 100644 --- a/examples/pipelines/finetune_stable_diffusion/pipeline.py +++ b/examples/pipelines/finetune_stable_diffusion/pipeline.py @@ -8,9 +8,7 @@ from pipeline_configs import PipelineConfigs from fondant.pipeline import ComponentOp, Pipeline, Client -from fondant.logger import configure_logging -configure_logging() logger = logging.getLogger(__name__) # General configs pipeline_name = "Test fondant pipeline" diff --git a/examples/pipelines/starcoder/pipeline.py b/examples/pipelines/starcoder/pipeline.py index 8a291c51e..98d542243 100644 --- a/examples/pipelines/starcoder/pipeline.py +++ b/examples/pipelines/starcoder/pipeline.py @@ -8,10 +8,8 @@ from pipeline_configs import PipelineConfigs from fondant.compiler import DockerCompiler -from fondant.logger import configure_logging from fondant.pipeline import ComponentOp, Pipeline -configure_logging() logger = logging.getLogger(__name__) diff --git a/fondant/__init__.py b/fondant/__init__.py index e69de29bb..0db5752a5 100644 --- a/fondant/__init__.py +++ b/fondant/__init__.py @@ -0,0 +1,5 @@ +import logging + +logging.basicConfig( + format="[%(asctime)s | %(name)s | %(levelname)s] %(message)s", level=logging.INFO +) diff --git a/fondant/logger.py b/fondant/logger.py deleted file mode 100644 index 08933eca2..000000000 --- a/fondant/logger.py +++ /dev/null @@ -1,23 +0,0 @@ -"""Utils file with useful methods.""" - -import logging -import os -import sys - -LOG_LEVEL = os.environ.get("LOG_LEVEL", default="INFO") - - -def configure_logging(log_level=LOG_LEVEL) -> None: - """Configure the root logger based on config settings.""" - logger = logging.getLogger() - - # set loglevel - level = logging.getLevelName(log_level) - logger.setLevel(level) - - # logging stdout handler - handler = logging.StreamHandler(stream=sys.stdout) - handler.setFormatter( - logging.Formatter(fmt="[%(asctime)s | %(name)s | %(levelname)s] %(message)s") - ) - logger.addHandler(handler) diff --git a/tests/test_logger.py b/tests/test_logger.py deleted file mode 100644 index 487c6b914..000000000 --- a/tests/test_logger.py +++ /dev/null @@ -1,25 +0,0 @@ -"""Test scripts for logger functionalities.""" -import logging - -import pytest - -from fondant.logger import configure_logging - - -@pytest.mark.parametrize( - "log_level, expected_level", - [ - ("DEBUG", logging.DEBUG), - ("INFO", logging.INFO), - ("WARNING", logging.WARNING), - ("ERROR", logging.ERROR), - ("CRITICAL", logging.CRITICAL), - ], -) -def test_configure_logging(log_level, expected_level): - """Function to test.""" - configure_logging(log_level) - - logger = logging.getLogger(__name__) - - assert logger.root.level == expected_level