From ca105940dd1ead9a6d49596fe3aa8ce5da84f84d Mon Sep 17 00:00:00 2001 From: Reid Morrison Date: Tue, 23 Jul 2024 10:50:04 -0400 Subject: [PATCH 1/2] Experimental support for Open telemetry logs --- lib/semantic_logger/appender.rb | 1 + .../appender/open_telemetry.rb | 100 ++++++++++++++++++ lib/semantic_logger/formatters.rb | 25 ++--- .../formatters/open_telemetry.rb | 18 ++++ 4 files changed, 132 insertions(+), 12 deletions(-) create mode 100644 lib/semantic_logger/appender/open_telemetry.rb create mode 100644 lib/semantic_logger/formatters/open_telemetry.rb diff --git a/lib/semantic_logger/appender.rb b/lib/semantic_logger/appender.rb index 6b950027..7003e6fc 100644 --- a/lib/semantic_logger/appender.rb +++ b/lib/semantic_logger/appender.rb @@ -17,6 +17,7 @@ module Appender autoload :MongoDB, "semantic_logger/appender/mongodb" autoload :NewRelic, "semantic_logger/appender/new_relic" autoload :NewRelicLogs, "semantic_logger/appender/new_relic_logs" + autoload :OpenTelemetry, "semantic_logger/appender/open_telemetry" autoload :Rabbitmq, "semantic_logger/appender/rabbitmq" autoload :Splunk, "semantic_logger/appender/splunk" autoload :SplunkHttp, "semantic_logger/appender/splunk_http" diff --git a/lib/semantic_logger/appender/open_telemetry.rb b/lib/semantic_logger/appender/open_telemetry.rb new file mode 100644 index 00000000..97aa44be --- /dev/null +++ b/lib/semantic_logger/appender/open_telemetry.rb @@ -0,0 +1,100 @@ +begin + require "opentelemetry/logs" +rescue LoadError + raise LoadError, 'Gem opentelemetry-logs-sdk is required for logging to Open Telemetry. Please add the gem "opentelemetry-logs-sdk" to your Gemfile.' +end + +# Open Telemetry Appender +# +# Writes log messages, and metrics to Open Telemetry. +# +module SemanticLogger + module Appender + class OpenTelemetry < SemanticLogger::Subscriber + attr_reader :name, :version, :logger + + CAPTURE_CONTEXT = ->(log) { log.set_context(:open_telemetry, ::OpenTelemetry::Context.current) } + + # Create a Open Telemetry Logger appender instance. + # + # Metric only log events are sent to the Open Telemetry Metrics API instead of the Logs API. + # I.e. A metric without a message or an exception. + # To disable this default behavior set `metrics: false` + # + # Example + # SemanticLogger.add_appender(appender: :open_telemetry) + def initialize(name: "SemanticLogger", + version: SemanticLogger::VERSION, + formatter: SemanticLogger::Formatters::OpenTelemetry.new, + metrics: true, + **args, + &block) + @name = name + @version = version + @logger = ::OpenTelemetry.logger_provider.logger(name: @name, version: @version) + + # Capture the current Open Telemetry context when a log entry is captured. + # Prevents duplicate subscribers as long as it is from a constant. + SemanticLogger.on_log(CAPTURE_CONTEXT) + + super(formatter: formatter, metrics: metrics, **args, &block) + end + + def log(log) + # return log_metric(log) if metrics && log.metric_only? + + ap formatter.call(log, self) + ap log.payload + + @logger.on_emit( + severity_text: log.level.to_s, + severity_number: severity_number(log.level), + timestamp: log.time, + body: formatter.call(log, self), + attributes: log.payload, + context: log.context[:open_telemetry] || ::OpenTelemetry::Context.current + ) + true + end + + # Flush all pending logs. + def flush + @logger.logger_provider.force_flush + end + + # Flush pending logs and close the appender + def close + @logger.logger_provider.shutdown + end + + private + + # For logging metrics only log events. + # def log_metric(log) + # puts "**** TODO: Metric Only Event ****" + # ap formatter.call(log, self) + # ap log.payload + # true + # end + + def severity_number(severity) + case severity.downcase + when :trace + ::OpenTelemetry::Logs::SeverityNumber::SEVERITY_NUMBER_TRACE + when :debug + ::OpenTelemetry::Logs::SeverityNumber::SEVERITY_NUMBER_DEBUG + when :info + ::OpenTelemetry::Logs::SeverityNumber::SEVERITY_NUMBER_INFO + when :warn + ::OpenTelemetry::Logs::SeverityNumber::SEVERITY_NUMBER_WARN + when :error + ::OpenTelemetry::Logs::SeverityNumber::SEVERITY_NUMBER_ERROR + when :fatal + ::OpenTelemetry::Logs::SeverityNumber::SEVERITY_NUMBER_FATAL + else + ::OpenTelemetry::Logs::SeverityNumber::SEVERITY_NUMBER_UNSPECIFIED + end + end + end + end +end diff --git a/lib/semantic_logger/formatters.rb b/lib/semantic_logger/formatters.rb index 95ef4b7c..5df9a068 100644 --- a/lib/semantic_logger/formatters.rb +++ b/lib/semantic_logger/formatters.rb @@ -1,17 +1,18 @@ module SemanticLogger module Formatters - autoload :Base, "semantic_logger/formatters/base" - autoload :Color, "semantic_logger/formatters/color" - autoload :Default, "semantic_logger/formatters/default" - autoload :Json, "semantic_logger/formatters/json" - autoload :Raw, "semantic_logger/formatters/raw" - autoload :OneLine, "semantic_logger/formatters/one_line" - autoload :Signalfx, "semantic_logger/formatters/signalfx" - autoload :Syslog, "semantic_logger/formatters/syslog" - autoload :Fluentd, "semantic_logger/formatters/fluentd" - autoload :Logfmt, "semantic_logger/formatters/logfmt" - autoload :SyslogCee, "semantic_logger/formatters/syslog_cee" - autoload :NewRelicLogs, "semantic_logger/formatters/new_relic_logs" + autoload :Base, "semantic_logger/formatters/base" + autoload :Color, "semantic_logger/formatters/color" + autoload :Default, "semantic_logger/formatters/default" + autoload :Json, "semantic_logger/formatters/json" + autoload :Raw, "semantic_logger/formatters/raw" + autoload :OneLine, "semantic_logger/formatters/one_line" + autoload :OpenTelemetry, "semantic_logger/formatters/open_telemetry" + autoload :Signalfx, "semantic_logger/formatters/signalfx" + autoload :Syslog, "semantic_logger/formatters/syslog" + autoload :Fluentd, "semantic_logger/formatters/fluentd" + autoload :Logfmt, "semantic_logger/formatters/logfmt" + autoload :SyslogCee, "semantic_logger/formatters/syslog_cee" + autoload :NewRelicLogs, "semantic_logger/formatters/new_relic_logs" # Return formatter that responds to call. # diff --git a/lib/semantic_logger/formatters/open_telemetry.rb b/lib/semantic_logger/formatters/open_telemetry.rb new file mode 100644 index 00000000..b49269ed --- /dev/null +++ b/lib/semantic_logger/formatters/open_telemetry.rb @@ -0,0 +1,18 @@ +require "json" +module SemanticLogger + module Formatters + class OpenTelemetry < Raw + # Remove the following fields since they are sent via the OpenTelemetry API + def time + end + + # Log level + def level + end + + # Payload is submitted directly as attributes + def payload + end + end + end +end From 6c912dd7d15fd053eb285c60c73a0d9f5025d5c8 Mon Sep 17 00:00:00 2001 From: Reid Morrison Date: Tue, 30 Jul 2024 10:18:18 -0400 Subject: [PATCH 2/2] Send keys as strings and move more attributes to the formatter --- .../appender/open_telemetry.rb | 36 ++++++------------- .../formatters/open_telemetry.rb | 30 +++++++++++++--- 2 files changed, 36 insertions(+), 30 deletions(-) diff --git a/lib/semantic_logger/appender/open_telemetry.rb b/lib/semantic_logger/appender/open_telemetry.rb index 97aa44be..38e6e73f 100644 --- a/lib/semantic_logger/appender/open_telemetry.rb +++ b/lib/semantic_logger/appender/open_telemetry.rb @@ -43,15 +43,18 @@ def initialize(name: "SemanticLogger", def log(log) # return log_metric(log) if metrics && log.metric_only? - ap formatter.call(log, self) - ap log.payload + body = formatter.call(log, self) + level = body.delete(:level) + level_index = body.delete(:level_index) + time = body.delete(:time) + payload = body.delete(:payload) @logger.on_emit( - severity_text: log.level.to_s, - severity_number: severity_number(log.level), - timestamp: log.time, - body: formatter.call(log, self), - attributes: log.payload, + severity_text: level, + severity_number: level_index, + timestamp: time, + body: body.transform_keys!(&:to_s), + attributes: payload, context: log.context[:open_telemetry] || ::OpenTelemetry::Context.current ) true @@ -76,25 +79,6 @@ def close # ap log.payload # true # end - - def severity_number(severity) - case severity.downcase - when :trace - ::OpenTelemetry::Logs::SeverityNumber::SEVERITY_NUMBER_TRACE - when :debug - ::OpenTelemetry::Logs::SeverityNumber::SEVERITY_NUMBER_DEBUG - when :info - ::OpenTelemetry::Logs::SeverityNumber::SEVERITY_NUMBER_INFO - when :warn - ::OpenTelemetry::Logs::SeverityNumber::SEVERITY_NUMBER_WARN - when :error - ::OpenTelemetry::Logs::SeverityNumber::SEVERITY_NUMBER_ERROR - when :fatal - ::OpenTelemetry::Logs::SeverityNumber::SEVERITY_NUMBER_FATAL - else - ::OpenTelemetry::Logs::SeverityNumber::SEVERITY_NUMBER_UNSPECIFIED - end - end end end end diff --git a/lib/semantic_logger/formatters/open_telemetry.rb b/lib/semantic_logger/formatters/open_telemetry.rb index b49269ed..1914994b 100644 --- a/lib/semantic_logger/formatters/open_telemetry.rb +++ b/lib/semantic_logger/formatters/open_telemetry.rb @@ -2,16 +2,38 @@ module SemanticLogger module Formatters class OpenTelemetry < Raw - # Remove the following fields since they are sent via the OpenTelemetry API - def time - end - # Log level def level + hash[:level] = log.level.to_s + hash[:level_index] = severity_number(log.level_index) end # Payload is submitted directly as attributes def payload + return unless log.payload.respond_to?(:empty?) && !log.payload.empty? + + hash[:payload] = log.payload.transform_keys!(&:to_s) + end + + private + + def severity_number(severity) + case severity + when :trace + ::OpenTelemetry::Logs::SeverityNumber::SEVERITY_NUMBER_TRACE + when :debug + ::OpenTelemetry::Logs::SeverityNumber::SEVERITY_NUMBER_DEBUG + when :info + ::OpenTelemetry::Logs::SeverityNumber::SEVERITY_NUMBER_INFO + when :warn + ::OpenTelemetry::Logs::SeverityNumber::SEVERITY_NUMBER_WARN + when :error + ::OpenTelemetry::Logs::SeverityNumber::SEVERITY_NUMBER_ERROR + when :fatal + ::OpenTelemetry::Logs::SeverityNumber::SEVERITY_NUMBER_FATAL + else + ::OpenTelemetry::Logs::SeverityNumber::SEVERITY_NUMBER_UNSPECIFIED + end end end end