From 599f8519c6b6ca4b7d89ffb970efe4dae660e341 Mon Sep 17 00:00:00 2001 From: Pablo Baeyens Date: Tue, 19 Dec 2023 17:39:29 +0100 Subject: [PATCH 1/3] [pkg/otlp/logs] Switch to Translator design --- pkg/otlp/logs/go.mod | 18 +- pkg/otlp/logs/go.sum | 43 ++- pkg/otlp/logs/logs_translator.go | 278 +++--------------- pkg/otlp/logs/transform.go | 265 +++++++++++++++++ ...s_translator_test.go => transform_test.go} | 0 5 files changed, 352 insertions(+), 252 deletions(-) create mode 100644 pkg/otlp/logs/transform.go rename pkg/otlp/logs/{logs_translator_test.go => transform_test.go} (100%) diff --git a/pkg/otlp/logs/go.mod b/pkg/otlp/logs/go.mod index 4ea2e363..2c88bd9b 100644 --- a/pkg/otlp/logs/go.mod +++ b/pkg/otlp/logs/go.mod @@ -6,22 +6,34 @@ require ( github.com/DataDog/datadog-api-client-go/v2 v2.13.0 github.com/DataDog/opentelemetry-mapping-go/pkg/otlp/attributes v0.10.0 github.com/stretchr/testify v1.8.4 + go.opentelemetry.io/collector/component v0.91.0 go.opentelemetry.io/collector/pdata v1.0.0 go.opentelemetry.io/collector/semconv v0.91.0 - go.uber.org/zap v1.24.0 + go.uber.org/zap v1.26.0 ) require ( github.com/DataDog/zstd v1.5.2 // indirect - github.com/benbjohnson/clock v1.1.0 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/golang/protobuf v1.5.3 // indirect + github.com/hashicorp/go-version v1.6.0 // indirect github.com/json-iterator/go v1.1.12 // indirect + github.com/knadh/koanf/maps v0.1.1 // indirect + github.com/knadh/koanf/providers/confmap v0.1.0 // indirect + github.com/knadh/koanf/v2 v2.0.1 // indirect + github.com/mitchellh/copystructure v1.2.0 // indirect + github.com/mitchellh/mapstructure v1.5.1-0.20220423185008-bf980b35cac4 // indirect + github.com/mitchellh/reflectwalk v1.0.2 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect - go.uber.org/atomic v1.7.0 // indirect + go.opentelemetry.io/collector/config/configtelemetry v0.91.0 // indirect + go.opentelemetry.io/collector/confmap v0.91.0 // indirect + go.opentelemetry.io/collector/featuregate v1.0.0 // indirect + go.opentelemetry.io/otel v1.21.0 // indirect + go.opentelemetry.io/otel/metric v1.21.0 // indirect + go.opentelemetry.io/otel/trace v1.21.0 // indirect go.uber.org/multierr v1.11.0 // indirect golang.org/x/net v0.18.0 // indirect golang.org/x/oauth2 v0.11.0 // indirect diff --git a/pkg/otlp/logs/go.sum b/pkg/otlp/logs/go.sum index d272d36a..11affd30 100644 --- a/pkg/otlp/logs/go.sum +++ b/pkg/otlp/logs/go.sum @@ -2,11 +2,11 @@ github.com/DataDog/datadog-api-client-go/v2 v2.13.0 h1:2c1dXSyUfum2YIVoYlqnBhV5J github.com/DataDog/datadog-api-client-go/v2 v2.13.0/go.mod h1:kntOqXEh1SmjwSDzW/eJkr9kS7EqttvEkelglWtJRbg= github.com/DataDog/zstd v1.5.2 h1:vUG4lAyuPCXO0TLbXvPv7EB7cNK1QV/luu55UHLrrn8= github.com/DataDog/zstd v1.5.2/go.mod h1:g4AWEaM3yOg3HYfnJ3YIawPnVdXJh9QME85blwSAmyw= -github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8= -github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/go-logr/logr v1.3.0 h1:2y3SDp0ZXuc6/cjLSZ+Q3ir+QB9T/iG5yYRXqsagWSY= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= @@ -14,20 +14,33 @@ github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaS github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/hashicorp/go-version v1.6.0 h1:feTTfFNnjP967rlCxM/I9g701jU+RN74YKx2mOkIeek= +github.com/hashicorp/go-version v1.6.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/knadh/koanf/maps v0.1.1 h1:G5TjmUh2D7G2YWf5SQQqSiHRJEjaicvU0KpypqB3NIs= +github.com/knadh/koanf/maps v0.1.1/go.mod h1:npD/QZY3V6ghQDdcQzl1W4ICNVTkohC8E73eI2xW4yI= +github.com/knadh/koanf/providers/confmap v0.1.0 h1:gOkxhHkemwG4LezxxN8DMOFopOPghxRVp7JbIvdvqzU= +github.com/knadh/koanf/providers/confmap v0.1.0/go.mod h1:2uLhxQzJnyHKfxG927awZC7+fyHFdQkd697K4MdLnIU= +github.com/knadh/koanf/v2 v2.0.1 h1:1dYGITt1I23x8cfx8ZnldtezdyaZtfAuRtIFOiRzK7g= +github.com/knadh/koanf/v2 v2.0.1/go.mod h1:ZeiIlIDXTE7w1lMT6UVcNiRAS2/rCeLn/GdLNvY1Dus= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/mitchellh/copystructure v1.2.0 h1:vpKXTN4ewci03Vljg/q9QvCGUDttBOGBIa15WveJJGw= +github.com/mitchellh/copystructure v1.2.0/go.mod h1:qLl+cE2AmVv+CoeAwDPye/v+N2HKCj9FbZEVFJRxO9s= +github.com/mitchellh/mapstructure v1.5.1-0.20220423185008-bf980b35cac4 h1:BpfhmLKZf+SjVanKKhCgf3bg+511DmU9eDQTen7LLbY= +github.com/mitchellh/mapstructure v1.5.1-0.20220423185008-bf980b35cac4/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= +github.com/mitchellh/reflectwalk v1.0.2 h1:G2LzWKi524PWgd3mLHV8Y5k7s6XUvT0Gef6zxSIeXaQ= +github.com/mitchellh/reflectwalk v1.0.2/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= -github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= @@ -37,17 +50,29 @@ github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcU github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +go.opentelemetry.io/collector/component v0.91.0 h1:aBT1i2zGyfh9PalYJLfXVvQp+osHyalwyDFselI1CtA= +go.opentelemetry.io/collector/component v0.91.0/go.mod h1:2KBHvjNFdU7oOjsObQeC4Ta2Ef607OISU5obznW00fw= +go.opentelemetry.io/collector/config/configtelemetry v0.91.0 h1:mEwvqrYfwUJ7LwYfpcF9M8z7LHFoYaKhEPhnERD/88E= +go.opentelemetry.io/collector/config/configtelemetry v0.91.0/go.mod h1:+LAXM5WFMW/UbTlAuSs6L/W72WC+q8TBJt/6z39FPOU= +go.opentelemetry.io/collector/confmap v0.91.0 h1:7U2MT+u74oEzq/WWrpXSLKB7nX5jPNC4drwtQdYfwKk= +go.opentelemetry.io/collector/confmap v0.91.0/go.mod h1:uxV+fZ85kG31oovL6Cl3fAMQ3RRPwUvfAbbA9WT1Yhk= +go.opentelemetry.io/collector/featuregate v1.0.0 h1:5MGqe2v5zxaoo73BUOvUTunftX5J8RGrbFsC2Ha7N3g= +go.opentelemetry.io/collector/featuregate v1.0.0/go.mod h1:xGbRuw+GbutRtVVSEy3YR2yuOlEyiUMhN2M9DJljgqY= go.opentelemetry.io/collector/pdata v1.0.0 h1:ECP2jnLztewsHmL1opL8BeMtWVc7/oSlKNhfY9jP8ec= go.opentelemetry.io/collector/pdata v1.0.0/go.mod h1:TsDFgs4JLNG7t6x9D8kGswXUz4mme+MyNChHx8zSF6k= go.opentelemetry.io/collector/semconv v0.91.0 h1:TRd+yDDfKQl+aNtS24wmEbJp1/QE/xAFV9SB5zWGxpE= go.opentelemetry.io/collector/semconv v0.91.0/go.mod h1:j/8THcqVxFna1FpvA2zYIsUperEtOaRaqoLYIN4doWw= -go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw= -go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= -go.uber.org/goleak v1.1.11 h1:wy28qYRKZgnJTxGxvye5/wgWr1EKjmUDGYox5mGlRlI= +go.opentelemetry.io/otel v1.21.0 h1:hzLeKBZEL7Okw2mGzZ0cc4k/A7Fta0uoPgaJCr8fsFc= +go.opentelemetry.io/otel v1.21.0/go.mod h1:QZzNPQPm1zLX4gZK4cMi+71eaorMSGT3A4znnUvNNEo= +go.opentelemetry.io/otel/metric v1.21.0 h1:tlYWfeo+Bocx5kLEloTjbcDwBuELRrIFxwdQ36PlJu4= +go.opentelemetry.io/otel/metric v1.21.0/go.mod h1:o1p3CA8nNHW8j5yuQLdc1eeqEaPfzug24uvsyIEJRWM= +go.opentelemetry.io/otel/trace v1.21.0 h1:WD9i5gzvoUPuXIXH24ZNBudiarZDKuekPqi/E8fpfLc= +go.opentelemetry.io/otel/trace v1.21.0/go.mod h1:LGbsEB0f9LGjN+OZaQQ26sohbOmiMR+BaslueVtS/qQ= +go.uber.org/goleak v1.2.0 h1:xqgm/S+aQvhWFTtR0XK3Jvg7z8kGV8P4X14IzwN3Eqk= go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= -go.uber.org/zap v1.24.0 h1:FiJd5l1UOLj0wCgbSE0rwwXHzEdAZS6hiiSnxJN/D60= -go.uber.org/zap v1.24.0/go.mod h1:2kMP+WWQ8aoFoedH3T2sq6iJ2yDWpHbP0f6MQbS9Gkg= +go.uber.org/zap v1.26.0 h1:sI7k6L95XOKS281NhVKOFCUNIvv9e0w4BF8N3u+tCRo= +go.uber.org/zap v1.26.0/go.mod h1:dtElttAiwGvoJ/vj4IwHBS/gXsEu/pZ50mUIRWuG0so= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= diff --git a/pkg/otlp/logs/logs_translator.go b/pkg/otlp/logs/logs_translator.go index 0d9f12a1..e6466b3f 100644 --- a/pkg/otlp/logs/logs_translator.go +++ b/pkg/otlp/logs/logs_translator.go @@ -15,251 +15,49 @@ package logs import ( - "encoding/binary" - "encoding/hex" - "strconv" - "strings" - - "github.com/DataDog/datadog-api-client-go/v2/api/datadog" "github.com/DataDog/datadog-api-client-go/v2/api/datadogV2" - "github.com/DataDog/opentelemetry-mapping-go/pkg/otlp/attributes" - "github.com/DataDog/opentelemetry-mapping-go/pkg/otlp/attributes/source" - "go.opentelemetry.io/collector/pdata/pcommon" + "go.opentelemetry.io/collector/component" "go.opentelemetry.io/collector/pdata/plog" - conventions "go.opentelemetry.io/collector/semconv/v1.6.1" - "go.uber.org/zap" -) - -const ( - // This set of constants specify the keys of the attributes that will be used to preserve - // the original OpenTelemetry logs attributes. - otelNamespace = "otel" - otelTraceID = otelNamespace + ".trace_id" - otelSpanID = otelNamespace + ".span_id" - otelSeverityNumber = otelNamespace + ".severity_number" - otelSeverityText = otelNamespace + ".severity_text" - otelTimestamp = otelNamespace + ".timestamp" -) -const ( - // This set of constants specify the keys of the attributes that will be used to represent Datadog - // counterparts to the OpenTelemetry Logs attributes. - ddNamespace = "dd" - ddTraceID = ddNamespace + ".trace_id" - ddSpanID = ddNamespace + ".span_id" - ddStatus = "status" - ddTimestamp = "@timestamp" ) -const ( - logLevelTrace = "trace" - logLevelDebug = "debug" - logLevelInfo = "info" - logLevelWarn = "warn" - logLevelError = "error" - logLevelFatal = "fatal" -) - -// Transform converts the log record in lr, which came in with the resource in res to a Datadog log item. -// the variable specifies if the log body should be sent as an attribute or as a plain message. -func Transform(lr plog.LogRecord, res pcommon.Resource, logger *zap.Logger) datadogV2.HTTPLogItem { - host, service := extractHostNameAndServiceName(res.Attributes(), lr.Attributes()) - - l := datadogV2.HTTPLogItem{ - AdditionalProperties: make(map[string]string), - } - if host != "" { - l.Hostname = datadog.PtrString(host) - } - if service != "" { - l.Service = datadog.PtrString(service) - } - - // we need to set log attributes as AdditionalProperties - // AdditionalProperties are treated as Datadog Log Attributes - var status string - lr.Attributes().Range(func(k string, v pcommon.Value) bool { - switch strings.ToLower(k) { - // set of remapping are taken from Datadog Backend - case "msg", "message", "log": - l.Message = v.AsString() - case "status", "severity", "level", "syslog.severity": - status = v.AsString() - case "traceid", "trace_id", "contextmap.traceid", "oteltraceid": - traceID, err := decodeTraceID(v.AsString()) - if err != nil { - logger.Warn("failed to decode trace id", - zap.String("trace_id", v.AsString()), - zap.Error(err)) - break - } - if l.AdditionalProperties[ddTraceID] == "" { - l.AdditionalProperties[ddTraceID] = strconv.FormatUint(traceIDToUint64(traceID), 10) - l.AdditionalProperties[otelTraceID] = v.AsString() - } - case "spanid", "span_id", "contextmap.spanid", "otelspanid": - spanID, err := decodeSpanID(v.AsString()) - if err != nil { - logger.Warn("failed to decode span id", - zap.String("span_id", v.AsString()), - zap.Error(err)) - break - } - if l.AdditionalProperties[ddSpanID] == "" { - l.AdditionalProperties[ddSpanID] = strconv.FormatUint(spanIDToUint64(spanID), 10) - l.AdditionalProperties[otelSpanID] = v.AsString() +// Translator of OTLP logs to Datadog format +type Translator struct { + set component.TelemetrySettings + otelTag string +} + +// NewTranslator returns a new Translator +func NewTranslator(set component.TelemetrySettings, otelSource string) (*Translator, error) { + return &Translator{ + set: set, + otelTag: "otel_source:" + otelSource, + }, nil +} + +// MapLogs from OTLP format to Datadog format. +func (t *Translator) MapLogs(ld plog.Logs) []datadogV2.HTTPLogItem { + rsl := ld.ResourceLogs() + var payloads []datadogV2.HTTPLogItem + for i := 0; i < rsl.Len(); i++ { + rl := rsl.At(i) + sls := rl.ScopeLogs() + res := rl.Resource() + for j := 0; j < sls.Len(); j++ { + sl := sls.At(j) + lsl := sl.LogRecords() + // iterate over Logs + for k := 0; k < lsl.Len(); k++ { + log := lsl.At(k) + payload := Transform(log, res, t.set.Logger) + ddtags := payload.GetDdtags() + if ddtags != "" { + payload.SetDdtags(ddtags + "," + t.otelTag) + } else { + payload.SetDdtags(t.otelTag) + } + payloads = append(payloads, payload) } - case "ddtags": - var tags = append(attributes.TagsFromAttributes(res.Attributes()), v.AsString()) - tagStr := strings.Join(tags, ",") - l.Ddtags = datadog.PtrString(tagStr) - default: - m := flattenAttribute(k, v, 1) - for k, v := range m { - l.AdditionalProperties[k] = v - } - } - return true - }) - res.Attributes().Range(func(k string, v pcommon.Value) bool { - // "hostname" and "service" are reserved keywords in HTTPLogItem - // Prefix the keys so they aren't overwritten when marshalling - if k == "hostname" || k == "service" { - l.AdditionalProperties["otel."+k] = v.AsString() - } else { - l.AdditionalProperties[k] = v.AsString() } - return true - }) - if traceID := lr.TraceID(); !traceID.IsEmpty() { - l.AdditionalProperties[ddTraceID] = strconv.FormatUint(traceIDToUint64(traceID), 10) - l.AdditionalProperties[otelTraceID] = hex.EncodeToString(traceID[:]) - } - if spanID := lr.SpanID(); !spanID.IsEmpty() { - l.AdditionalProperties[ddSpanID] = strconv.FormatUint(spanIDToUint64(spanID), 10) - l.AdditionalProperties[otelSpanID] = hex.EncodeToString(spanID[:]) - } - - // we want to use the serverity that client has set on the log and let Datadog backend - // decide the appropriate level - if lr.SeverityText() != "" { - if status == "" { - status = lr.SeverityText() - } - l.AdditionalProperties[otelSeverityText] = lr.SeverityText() - } - if lr.SeverityNumber() != 0 { - if status == "" { - status = statusFromSeverityNumber(lr.SeverityNumber()) - } - l.AdditionalProperties[otelSeverityNumber] = strconv.Itoa(int(lr.SeverityNumber())) - } - l.AdditionalProperties[ddStatus] = status - // for Datadog to use the same timestamp we need to set the additional property of "@timestamp" - if lr.Timestamp() != 0 { - // we are retaining the nano second precision in this property - l.AdditionalProperties[otelTimestamp] = strconv.FormatInt(lr.Timestamp().AsTime().UnixNano(), 10) - l.AdditionalProperties[ddTimestamp] = lr.Timestamp().AsTime().Format("2006-01-02T15:04:05.000Z07:00") - } - if l.Message == "" { - // set the Message to the Body in case it wasn't already parsed as part of the attributes - l.Message = lr.Body().AsString() - } - - if !l.HasDdtags() { - var tags = attributes.TagsFromAttributes(res.Attributes()) - tagStr := strings.Join(tags, ",") - l.Ddtags = datadog.PtrString(tagStr) - } - - return l -} - -func flattenAttribute(key string, val pcommon.Value, depth int) map[string]string { - result := make(map[string]string) - - if val.Type() != pcommon.ValueTypeMap || depth == 10 { - result[key] = val.AsString() - return result - } - - val.Map().Range(func(k string, v pcommon.Value) bool { - newKey := key + "." + k - nestedResult := flattenAttribute(newKey, v, depth+1) - for nk, nv := range nestedResult { - result[nk] = nv - } - return true - }) - - return result -} - -func extractHostNameAndServiceName(resourceAttrs pcommon.Map, logAttrs pcommon.Map) (host string, service string) { - if src, ok := attributes.SourceFromAttrs(resourceAttrs); ok && src.Kind == source.HostnameKind { - host = src.Identifier - } - // hostName is blank from resource - // we need to derive from log attributes - if host == "" { - if src, ok := attributes.SourceFromAttrs(logAttrs); ok && src.Kind == source.HostnameKind { - host = src.Identifier - } - } - if s, ok := resourceAttrs.Get(conventions.AttributeServiceName); ok { - service = s.AsString() - } - // serviceName is blank from resource - // we need to derive from log attributes - if service == "" { - if s, ok := logAttrs.Get(conventions.AttributeServiceName); ok { - service = s.AsString() - } - } - return host, service -} - -func decodeTraceID(traceID string) ([16]byte, error) { - var ret [16]byte - _, err := hex.Decode(ret[:], []byte(traceID)) - return ret, err -} - -func decodeSpanID(spanID string) ([8]byte, error) { - var ret [8]byte - _, err := hex.Decode(ret[:], []byte(spanID)) - return ret, err -} - -// traceIDToUint64 converts 128bit traceId to 64 bit uint64 -func traceIDToUint64(b [16]byte) uint64 { - return binary.BigEndian.Uint64(b[len(b)-8:]) -} - -// spanIDToUint64 converts byte array to uint64 -func spanIDToUint64(b [8]byte) uint64 { - return binary.BigEndian.Uint64(b[:]) -} - -// statusFromSeverityNumber converts the severity number to log level -// https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/logs/data-model.md#field-severitynumber -// this is not exactly datadog log levels , but derived from range name from above link -// see https://docs.datadoghq.com/logs/log_configuration/processors/?tab=ui#log-status-remapper for details on how it maps to datadog level -func statusFromSeverityNumber(severity plog.SeverityNumber) string { - switch { - case severity <= 4: - return logLevelTrace - case severity <= 8: - return logLevelDebug - case severity <= 12: - return logLevelInfo - case severity <= 16: - return logLevelWarn - case severity <= 20: - return logLevelError - case severity <= 24: - return logLevelFatal - default: - // By default, treat this as error - return logLevelError } + return payloads } diff --git a/pkg/otlp/logs/transform.go b/pkg/otlp/logs/transform.go new file mode 100644 index 00000000..1c4ab47d --- /dev/null +++ b/pkg/otlp/logs/transform.go @@ -0,0 +1,265 @@ +// Copyright The OpenTelemetry Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package logs + +import ( + "encoding/binary" + "encoding/hex" + "strconv" + "strings" + + "github.com/DataDog/datadog-api-client-go/v2/api/datadog" + "github.com/DataDog/datadog-api-client-go/v2/api/datadogV2" + "github.com/DataDog/opentelemetry-mapping-go/pkg/otlp/attributes" + "github.com/DataDog/opentelemetry-mapping-go/pkg/otlp/attributes/source" + "go.opentelemetry.io/collector/pdata/pcommon" + "go.opentelemetry.io/collector/pdata/plog" + conventions "go.opentelemetry.io/collector/semconv/v1.6.1" + "go.uber.org/zap" +) + +const ( + // This set of constants specify the keys of the attributes that will be used to preserve + // the original OpenTelemetry logs attributes. + otelNamespace = "otel" + otelTraceID = otelNamespace + ".trace_id" + otelSpanID = otelNamespace + ".span_id" + otelSeverityNumber = otelNamespace + ".severity_number" + otelSeverityText = otelNamespace + ".severity_text" + otelTimestamp = otelNamespace + ".timestamp" +) +const ( + // This set of constants specify the keys of the attributes that will be used to represent Datadog + // counterparts to the OpenTelemetry Logs attributes. + ddNamespace = "dd" + ddTraceID = ddNamespace + ".trace_id" + ddSpanID = ddNamespace + ".span_id" + ddStatus = "status" + ddTimestamp = "@timestamp" +) + +const ( + logLevelTrace = "trace" + logLevelDebug = "debug" + logLevelInfo = "info" + logLevelWarn = "warn" + logLevelError = "error" + logLevelFatal = "fatal" +) + +// Transform converts the log record in lr, which came in with the resource in res to a Datadog log item. +// the variable specifies if the log body should be sent as an attribute or as a plain message. +// Deprecated: use Translator instead. +func Transform(lr plog.LogRecord, res pcommon.Resource, logger *zap.Logger) datadogV2.HTTPLogItem { + host, service := extractHostNameAndServiceName(res.Attributes(), lr.Attributes()) + l := datadogV2.HTTPLogItem{ + AdditionalProperties: make(map[string]string), + } + if host != "" { + l.Hostname = datadog.PtrString(host) + } + if service != "" { + l.Service = datadog.PtrString(service) + } + + // we need to set log attributes as AdditionalProperties + // AdditionalProperties are treated as Datadog Log Attributes + var status string + lr.Attributes().Range(func(k string, v pcommon.Value) bool { + switch strings.ToLower(k) { + // set of remapping are taken from Datadog Backend + case "msg", "message", "log": + l.Message = v.AsString() + case "status", "severity", "level", "syslog.severity": + status = v.AsString() + case "traceid", "trace_id", "contextmap.traceid", "oteltraceid": + traceID, err := decodeTraceID(v.AsString()) + if err != nil { + logger.Warn("failed to decode trace id", + zap.String("trace_id", v.AsString()), + zap.Error(err)) + break + } + if l.AdditionalProperties[ddTraceID] == "" { + l.AdditionalProperties[ddTraceID] = strconv.FormatUint(traceIDToUint64(traceID), 10) + l.AdditionalProperties[otelTraceID] = v.AsString() + } + case "spanid", "span_id", "contextmap.spanid", "otelspanid": + spanID, err := decodeSpanID(v.AsString()) + if err != nil { + logger.Warn("failed to decode span id", + zap.String("span_id", v.AsString()), + zap.Error(err)) + break + } + if l.AdditionalProperties[ddSpanID] == "" { + l.AdditionalProperties[ddSpanID] = strconv.FormatUint(spanIDToUint64(spanID), 10) + l.AdditionalProperties[otelSpanID] = v.AsString() + } + case "ddtags": + var tags = append(attributes.TagsFromAttributes(res.Attributes()), v.AsString()) + tagStr := strings.Join(tags, ",") + l.Ddtags = datadog.PtrString(tagStr) + default: + m := flattenAttribute(k, v, 1) + for k, v := range m { + l.AdditionalProperties[k] = v + } + } + return true + }) + res.Attributes().Range(func(k string, v pcommon.Value) bool { + // "hostname" and "service" are reserved keywords in HTTPLogItem + // Prefix the keys so they aren't overwritten when marshalling + if k == "hostname" || k == "service" { + l.AdditionalProperties["otel."+k] = v.AsString() + } else { + l.AdditionalProperties[k] = v.AsString() + } + return true + }) + if traceID := lr.TraceID(); !traceID.IsEmpty() { + l.AdditionalProperties[ddTraceID] = strconv.FormatUint(traceIDToUint64(traceID), 10) + l.AdditionalProperties[otelTraceID] = hex.EncodeToString(traceID[:]) + } + if spanID := lr.SpanID(); !spanID.IsEmpty() { + l.AdditionalProperties[ddSpanID] = strconv.FormatUint(spanIDToUint64(spanID), 10) + l.AdditionalProperties[otelSpanID] = hex.EncodeToString(spanID[:]) + } + + // we want to use the serverity that client has set on the log and let Datadog backend + // decide the appropriate level + if lr.SeverityText() != "" { + if status == "" { + status = lr.SeverityText() + } + l.AdditionalProperties[otelSeverityText] = lr.SeverityText() + } + if lr.SeverityNumber() != 0 { + if status == "" { + status = statusFromSeverityNumber(lr.SeverityNumber()) + } + l.AdditionalProperties[otelSeverityNumber] = strconv.Itoa(int(lr.SeverityNumber())) + } + l.AdditionalProperties[ddStatus] = status + // for Datadog to use the same timestamp we need to set the additional property of "@timestamp" + if lr.Timestamp() != 0 { + // we are retaining the nano second precision in this property + l.AdditionalProperties[otelTimestamp] = strconv.FormatInt(lr.Timestamp().AsTime().UnixNano(), 10) + l.AdditionalProperties[ddTimestamp] = lr.Timestamp().AsTime().Format("2006-01-02T15:04:05.000Z07:00") + } + if l.Message == "" { + // set the Message to the Body in case it wasn't already parsed as part of the attributes + l.Message = lr.Body().AsString() + } + + if !l.HasDdtags() { + var tags = attributes.TagsFromAttributes(res.Attributes()) + tagStr := strings.Join(tags, ",") + l.Ddtags = datadog.PtrString(tagStr) + } + + return l +} + +func flattenAttribute(key string, val pcommon.Value, depth int) map[string]string { + result := make(map[string]string) + + if val.Type() != pcommon.ValueTypeMap || depth == 10 { + result[key] = val.AsString() + return result + } + + val.Map().Range(func(k string, v pcommon.Value) bool { + newKey := key + "." + k + nestedResult := flattenAttribute(newKey, v, depth+1) + for nk, nv := range nestedResult { + result[nk] = nv + } + return true + }) + + return result +} + +func extractHostNameAndServiceName(resourceAttrs pcommon.Map, logAttrs pcommon.Map) (host string, service string) { + if src, ok := attributes.SourceFromAttrs(resourceAttrs); ok && src.Kind == source.HostnameKind { + host = src.Identifier + } + // hostName is blank from resource + // we need to derive from log attributes + if host == "" { + if src, ok := attributes.SourceFromAttrs(logAttrs); ok && src.Kind == source.HostnameKind { + host = src.Identifier + } + } + if s, ok := resourceAttrs.Get(conventions.AttributeServiceName); ok { + service = s.AsString() + } + // serviceName is blank from resource + // we need to derive from log attributes + if service == "" { + if s, ok := logAttrs.Get(conventions.AttributeServiceName); ok { + service = s.AsString() + } + } + return host, service +} + +func decodeTraceID(traceID string) ([16]byte, error) { + var ret [16]byte + _, err := hex.Decode(ret[:], []byte(traceID)) + return ret, err +} + +func decodeSpanID(spanID string) ([8]byte, error) { + var ret [8]byte + _, err := hex.Decode(ret[:], []byte(spanID)) + return ret, err +} + +// traceIDToUint64 converts 128bit traceId to 64 bit uint64 +func traceIDToUint64(b [16]byte) uint64 { + return binary.BigEndian.Uint64(b[len(b)-8:]) +} + +// spanIDToUint64 converts byte array to uint64 +func spanIDToUint64(b [8]byte) uint64 { + return binary.BigEndian.Uint64(b[:]) +} + +// statusFromSeverityNumber converts the severity number to log level +// https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/logs/data-model.md#field-severitynumber +// this is not exactly datadog log levels , but derived from range name from above link +// see https://docs.datadoghq.com/logs/log_configuration/processors/?tab=ui#log-status-remapper for details on how it maps to datadog level +func statusFromSeverityNumber(severity plog.SeverityNumber) string { + switch { + case severity <= 4: + return logLevelTrace + case severity <= 8: + return logLevelDebug + case severity <= 12: + return logLevelInfo + case severity <= 16: + return logLevelWarn + case severity <= 20: + return logLevelError + case severity <= 24: + return logLevelFatal + default: + // By default, treat this as error + return logLevelError + } +} diff --git a/pkg/otlp/logs/logs_translator_test.go b/pkg/otlp/logs/transform_test.go similarity index 100% rename from pkg/otlp/logs/logs_translator_test.go rename to pkg/otlp/logs/transform_test.go From c0b8610b820b56d2687bbf6289fa7066f8fb6951 Mon Sep 17 00:00:00 2001 From: Pablo Baeyens Date: Wed, 20 Dec 2023 11:39:03 +0100 Subject: [PATCH 2/3] Address feedback --- pkg/otlp/logs/{logs_translator.go => translator.go} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename pkg/otlp/logs/{logs_translator.go => translator.go} (100%) diff --git a/pkg/otlp/logs/logs_translator.go b/pkg/otlp/logs/translator.go similarity index 100% rename from pkg/otlp/logs/logs_translator.go rename to pkg/otlp/logs/translator.go From 3688db9c0301430b5ce66d97cbc6f93d422f44bf Mon Sep 17 00:00:00 2001 From: Pablo Baeyens Date: Wed, 20 Dec 2023 11:40:58 +0100 Subject: [PATCH 3/3] Add changelog note --- .chloggen/mx-psi_attributes-translator.yaml | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) create mode 100755 .chloggen/mx-psi_attributes-translator.yaml diff --git a/.chloggen/mx-psi_attributes-translator.yaml b/.chloggen/mx-psi_attributes-translator.yaml new file mode 100755 index 00000000..6ace6f4c --- /dev/null +++ b/.chloggen/mx-psi_attributes-translator.yaml @@ -0,0 +1,16 @@ +# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix' +change_type: deprecation + +# The name of the component (e.g. pkg/quantile) +component: pkg/otlp/logs + +# A brief description of the change. Surround your text with quotes ("") if it needs to start with a backtick (`). +note: Deprecate `logs.Transform` in favor of `logs.Translator` + +# The PR related to this change +issues: [230] + +# (Optional) One or more lines of additional information to render under the primary note. +# These lines will be padded with 2 spaces and then inserted directly into the document. +# Use pipe (|) for multiline entries. +subtext: