From 0c7f06018bf41c6609ce3742c9ea982e2ce0aa5a Mon Sep 17 00:00:00 2001 From: vitaliizazmic <75620293+vitaliizazmic@users.noreply.github.com> Date: Fri, 22 Oct 2021 17:14:01 +0300 Subject: [PATCH] =?UTF-8?q?=F0=9F=90=9B=20Airbyte=20CDK:=20transforming=20?= =?UTF-8?q?Python=20log=20levels=20to=20Airbyte=20protocol=20log=20levels?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Airbyte CDK native logger #1279 - transforming Python log levels to Airbyte protocol log levels * Airbyte CDK native logger #1279 - add test for level types transforming * CDK standard logger #1279 - test for transform critical level to fatal --- airbyte-cdk/python/CHANGELOG.md | 3 +++ airbyte-cdk/python/airbyte_cdk/logger.py | 13 ++++++++++++- .../airbyte_cdk/models/airbyte_protocol.py | 2 -- airbyte-cdk/python/setup.py | 2 +- airbyte-cdk/python/unit_tests/test_logger.py | 18 ++++++++++++++++++ .../models/airbyte_protocol.py | 2 -- .../airbyte_protocol/airbyte_protocol.yaml | 2 -- 7 files changed, 34 insertions(+), 8 deletions(-) diff --git a/airbyte-cdk/python/CHANGELOG.md b/airbyte-cdk/python/CHANGELOG.md index 27dbd908c3be..33f56cdb0104 100644 --- a/airbyte-cdk/python/CHANGELOG.md +++ b/airbyte-cdk/python/CHANGELOG.md @@ -1,5 +1,8 @@ # Changelog +## 0.1.31 +Transforming Python log levels to Airbyte protocol log levels + ## 0.1.30 Updated OAuth2Specification.rootObject type in airbyte_protocol to allow string or int diff --git a/airbyte-cdk/python/airbyte_cdk/logger.py b/airbyte-cdk/python/airbyte_cdk/logger.py index 9db5dec00c79..22dda86f5b3d 100644 --- a/airbyte-cdk/python/airbyte_cdk/logger.py +++ b/airbyte-cdk/python/airbyte_cdk/logger.py @@ -42,10 +42,21 @@ def init_logger(name: str = None): class AirbyteLogFormatter(logging.Formatter): """Output log records using AirbyteMessage""" + # Transforming Python log levels to Airbyte protocol log levels + level_mapping = { + logging.FATAL: "FATAL", + logging.ERROR: "ERROR", + logging.WARNING: "WARN", + logging.INFO: "INFO", + logging.DEBUG: "DEBUG", + TRACE_LEVEL_NUM: "TRACE", + } + def format(self, record: logging.LogRecord) -> str: """Return a JSON representation of the log message""" message = super().format(record) - log_message = AirbyteMessage(type="LOG", log=AirbyteLogMessage(level=record.levelname, message=message)) + airbyte_level = self.level_mapping.get(record.levelno, "INFO") + log_message = AirbyteMessage(type="LOG", log=AirbyteLogMessage(level=airbyte_level, message=message)) return log_message.json(exclude_unset=True) diff --git a/airbyte-cdk/python/airbyte_cdk/models/airbyte_protocol.py b/airbyte-cdk/python/airbyte_cdk/models/airbyte_protocol.py index de14f3043ae9..0e7477a838ff 100644 --- a/airbyte-cdk/python/airbyte_cdk/models/airbyte_protocol.py +++ b/airbyte-cdk/python/airbyte_cdk/models/airbyte_protocol.py @@ -44,10 +44,8 @@ class Config: class Level(Enum): FATAL = "FATAL" - CRITICAL = "CRITICAL" ERROR = "ERROR" WARN = "WARN" - WARNING = "WARNING" INFO = "INFO" DEBUG = "DEBUG" TRACE = "TRACE" diff --git a/airbyte-cdk/python/setup.py b/airbyte-cdk/python/setup.py index 4570265c97bf..392dc7d22fe1 100644 --- a/airbyte-cdk/python/setup.py +++ b/airbyte-cdk/python/setup.py @@ -15,7 +15,7 @@ setup( name="airbyte-cdk", - version="0.1.30", + version="0.1.31", description="A framework for writing Airbyte Connectors.", long_description=README, long_description_content_type="text/markdown", diff --git a/airbyte-cdk/python/unit_tests/test_logger.py b/airbyte-cdk/python/unit_tests/test_logger.py index 526c46eaaa93..63a29305a5dd 100644 --- a/airbyte-cdk/python/unit_tests/test_logger.py +++ b/airbyte-cdk/python/unit_tests/test_logger.py @@ -31,6 +31,24 @@ def test_formatter(logger, caplog): assert message == "Test formatter" +def test_level_transform(logger, caplog): + formatter = AirbyteLogFormatter() + logger.warning("Test level transform warn") + logger.critical("Test level transform critical") + record_warn = caplog.records[0] + record_critical = caplog.records[1] + formatted_record_warn = formatter.format(record_warn) + formatted_record_warn_data = json.loads(formatted_record_warn) + log_warn = formatted_record_warn_data.get("log") + level_warn = log_warn.get("level") + formatted_record_critical = formatter.format(record_critical) + formatted_record_critical_data = json.loads(formatted_record_critical) + log_critical = formatted_record_critical_data.get("log") + level_critical = log_critical.get("level") + assert level_warn == "WARN" + assert level_critical == "FATAL" + + def test_trace(logger, caplog): logger.trace("Test trace 1") record = caplog.records[0] diff --git a/airbyte-integrations/bases/airbyte-protocol/airbyte_protocol/models/airbyte_protocol.py b/airbyte-integrations/bases/airbyte-protocol/airbyte_protocol/models/airbyte_protocol.py index de14f3043ae9..0e7477a838ff 100644 --- a/airbyte-integrations/bases/airbyte-protocol/airbyte_protocol/models/airbyte_protocol.py +++ b/airbyte-integrations/bases/airbyte-protocol/airbyte_protocol/models/airbyte_protocol.py @@ -44,10 +44,8 @@ class Config: class Level(Enum): FATAL = "FATAL" - CRITICAL = "CRITICAL" ERROR = "ERROR" WARN = "WARN" - WARNING = "WARNING" INFO = "INFO" DEBUG = "DEBUG" TRACE = "TRACE" diff --git a/airbyte-protocol/models/src/main/resources/airbyte_protocol/airbyte_protocol.yaml b/airbyte-protocol/models/src/main/resources/airbyte_protocol/airbyte_protocol.yaml index 8b99b171028d..e621be83553c 100644 --- a/airbyte-protocol/models/src/main/resources/airbyte_protocol/airbyte_protocol.yaml +++ b/airbyte-protocol/models/src/main/resources/airbyte_protocol/airbyte_protocol.yaml @@ -86,10 +86,8 @@ definitions: type: string enum: - FATAL - - CRITICAL - ERROR - WARN - - WARNING - INFO - DEBUG - TRACE