Strip leading 0s from traceIds and spanIds if the span source is Jaeg…

…er (#3849) (#3855) * traceId has 16 bytes, spanId and parentSpanId have 8 bytes * Jaeger exports traceId, spantId and parentSpanId without leading 0s. We should do the same if the span source is Jaeger in order to enable log correlation (#3837) * Extracted Jaeger traceId/spanId formatting to their own functions (#3837) * Added Jaeger traceId/spanId formatting change to changelog (#3837) * Fix golint (#3837) Co-authored-by: Tobias Stadler <[email protected]>
elastic · Jun 8, 2020 · 410447f · 410447f
1 parent a2ee4cc
commit 410447f
Show file tree

Hide file tree

Showing 10 changed files with 146 additions and 57 deletions.
diff --git a/processor/otel/consumer.go b/processor/otel/consumer.go
@@ -30,6 +30,7 @@ import (
 	tracepb "github.com/census-instrumentation/opencensus-proto/gen-go/trace/v1"
 	"github.com/golang/protobuf/ptypes/timestamp"
 	"github.com/open-telemetry/opentelemetry-collector/consumer/consumerdata"
+	tracetranslator "github.com/open-telemetry/opentelemetry-collector/translator/trace"
 
 	"github.com/elastic/beats/v7/libbeat/common"
 	"github.com/elastic/beats/v7/libbeat/logp"
@@ -81,14 +82,25 @@ func (c *Consumer) convert(td consumerdata.TraceData) *model.Batch {
 			continue
 		}
 
-		var parentID *string
 		root := len(otelSpan.ParentSpanId) == 0
-		if !root {
-			str := fmt.Sprintf("%x", otelSpan.ParentSpanId)
-			parentID = &str
+
+		var parentID, spanID, traceID string
+		if td.SourceFormat == sourceFormatJaeger {
+			if !root {
+				parentID = formatJaegerSpanID(otelSpan.ParentSpanId)
+			}
+
+			traceID = formatJaegerTraceID(otelSpan.TraceId)
+			spanID = formatJaegerSpanID(otelSpan.SpanId)
+		} else {
+			if !root {
+				parentID = fmt.Sprintf("%x", otelSpan.ParentSpanId)
+			}
+
+			traceID = fmt.Sprintf("%x", otelSpan.TraceId)
+			spanID = fmt.Sprintf("%x", otelSpan.SpanId)
 		}
-		traceID := fmt.Sprintf("%x", otelSpan.TraceId)
-		spanID := fmt.Sprintf("%x", otelSpan.SpanId)
+
 		startTime := parseTimestamp(otelSpan.StartTime)
 		var duration float64
 		if otelSpan.EndTime != nil && !startTime.IsZero() {
@@ -99,7 +111,7 @@ func (c *Consumer) convert(td consumerdata.TraceData) *model.Batch {
 			transaction := model.Transaction{
 				Metadata:  md,
 				ID:        spanID,
-				ParentID:  parentID,
+				ParentID:  &parentID,
 				TraceID:   traceID,
 				Timestamp: startTime,
 				Duration:  duration,
@@ -116,7 +128,7 @@ func (c *Consumer) convert(td consumerdata.TraceData) *model.Batch {
 			span := model.Span{
 				Metadata:  md,
 				ID:        spanID,
-				ParentID:  parentID,
+				ParentID:  &parentID,
 				TraceID:   &traceID,
 				Timestamp: startTime,
 				Duration:  duration,
@@ -591,3 +603,27 @@ func truncate(s string) string {
 	}
 	return s
 }
+
+// formatJaegerTraceID returns the traceID as string in Jaeger format (hexadecimal without leading zeros)
+func formatJaegerTraceID(traceID []byte) string {
+	jaegerTraceIDHigh, jaegerTraceIDLow, err := tracetranslator.BytesToUInt64TraceID(traceID)
+	if err != nil {
+		return fmt.Sprintf("%x", traceID)
+	}
+
+	if jaegerTraceIDHigh == 0 {
+		return fmt.Sprintf("%x", jaegerTraceIDLow)
+	}
+
+	return fmt.Sprintf("%x%016x", jaegerTraceIDHigh, jaegerTraceIDLow)
+}
+
+// formatJaegerSpanID returns the spanID as string in Jaeger format (hexadecimal without leading zeros)
+func formatJaegerSpanID(spanID []byte) string {
+	jaegerSpanID, err := tracetranslator.BytesToUInt64SpanID(spanID)
+	if err != nil {
+		return fmt.Sprintf("%x", spanID)
+	}
+
+	return fmt.Sprintf("%x", jaegerSpanID)
+}
diff --git a/processor/otel/consumer_test.go b/processor/otel/consumer_test.go
@@ -52,7 +52,7 @@ func TestConsumer_ConsumeTraceData(t *testing.T) {
 			SourceFormat: "jaeger",
 			Spans: []*tracepb.Span{
 				{Kind: tracepb.Span_SERVER, StartTime: &timestamp.Timestamp{Seconds: 1576500418, Nanos: 768068}},
-				{ParentSpanId: []byte("FF0X"), StartTime: &timestamp.Timestamp{Seconds: 1576500418, Nanos: 768068}},
+				{ParentSpanId: []byte{0, 0, 0, 0, 70, 70, 48, 88}, StartTime: &timestamp.Timestamp{Seconds: 1576500418, Nanos: 768068}},
 			}}},
 	} {
 		t.Run(tc.name, func(t *testing.T) {
@@ -69,8 +69,8 @@ func TestConsumer_ConsumeTraceData(t *testing.T) {
 
 func TestConsumer_Metadata(t *testing.T) {
 	spans := []*tracepb.Span{{
-		TraceId:   []byte("FFx0"),
-		SpanId:    []byte("AAFF"),
+		TraceId:   []byte{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 70, 70, 120, 48},
+		SpanId:    []byte{0, 0, 0, 0, 65, 65, 70, 70},
 		Kind:      tracepb.Span_CLIENT,
 		StartTime: testStartTime(),
 	}}
@@ -121,6 +121,22 @@ func TestConsumer_Metadata(t *testing.T) {
 				ServiceInfo: &commonpb.ServiceInfo{},
 			},
 		},
+	}, {
+		name: "jaeger_full-traceid",
+		td: consumerdata.TraceData{
+			SourceFormat: "jaeger",
+			Spans: []*tracepb.Span{{
+				TraceId:   []byte{0, 0, 0, 0, 70, 70, 120, 48, 0, 0, 0, 0, 70, 70, 120, 48},
+				SpanId:    []byte{0, 0, 0, 0, 65, 65, 70, 70},
+				Kind:      tracepb.Span_CLIENT,
+				StartTime: testStartTime(),
+			}},
+			Node: &commonpb.Node{
+				Identifier:  &commonpb.ProcessIdentifier{},
+				LibraryInfo: &commonpb.LibraryInfo{},
+				ServiceInfo: &commonpb.ServiceInfo{},
+			},
+		},
 	}, {
 		name: "minimal",
 		td:   consumerdata.TraceData{SourceFormat: "foo", Spans: spans},
@@ -146,8 +162,8 @@ func TestConsumer_Transaction(t *testing.T) {
 			td: consumerdata.TraceData{SourceFormat: "jaeger",
 				Node: &commonpb.Node{Identifier: &commonpb.ProcessIdentifier{HostName: "host-abc"}},
 				Spans: []*tracepb.Span{{
-					TraceId:                 []byte("FFx0"),
-					SpanId:                  []byte("AAFF"),
+					TraceId:                 []byte{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 70, 70, 120, 48},
+					SpanId:                  []byte{0, 0, 0, 0, 65, 65, 70, 70},
 					StartTime:               testStartTime(),
 					EndTime:                 testEndTime(),
 					Name:                    testTruncatableString("HTTP GET"),
@@ -173,7 +189,7 @@ func TestConsumer_Transaction(t *testing.T) {
 			td: consumerdata.TraceData{SourceFormat: "jaeger",
 				Node: &commonpb.Node{Identifier: &commonpb.ProcessIdentifier{HostName: "host-abc"}},
 				Spans: []*tracepb.Span{{
-					ParentSpanId: []byte("abcd"), Kind: tracepb.Span_SERVER,
+					ParentSpanId: []byte{0, 0, 0, 0, 97, 98, 99, 100}, Kind: tracepb.Span_SERVER,
 					StartTime: testStartTime(),
 					Attributes: &tracepb.Span_Attributes{AttributeMap: map[string]*tracepb.AttributeValue{
 						"http.status_code": testAttributeIntValue(200),
@@ -184,7 +200,7 @@ func TestConsumer_Transaction(t *testing.T) {
 			td: consumerdata.TraceData{SourceFormat: "jaeger",
 				Node: &commonpb.Node{Identifier: &commonpb.ProcessIdentifier{HostName: "host-abc"}},
 				Spans: []*tracepb.Span{{
-					ParentSpanId: []byte("abcd"), Kind: tracepb.Span_SERVER,
+					ParentSpanId: []byte{0, 0, 0, 0, 97, 98, 99, 100}, Kind: tracepb.Span_SERVER,
 					StartTime: testStartTime(),
 					Status:    &tracepb.Status{Code: 200},
 					Attributes: &tracepb.Span_Attributes{AttributeMap: map[string]*tracepb.AttributeValue{
@@ -239,7 +255,7 @@ func TestConsumer_Span(t *testing.T) {
 			td: consumerdata.TraceData{SourceFormat: "jaeger",
 				Node: &commonpb.Node{Identifier: &commonpb.ProcessIdentifier{HostName: "host-abc"}},
 				Spans: []*tracepb.Span{{
-					TraceId: []byte("FFx0"), SpanId: []byte("AAFF"), ParentSpanId: []byte("XXXX"),
+					TraceId: []byte{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 70, 70, 120, 48}, SpanId: []byte{0, 0, 0, 0, 65, 65, 70, 70}, ParentSpanId: []byte{0, 0, 0, 0, 88, 88, 88, 88},
 					StartTime: testStartTime(), EndTime: testEndTime(),
 					Name: testTruncatableString("HTTP GET"),
 					Attributes: &tracepb.Span_Attributes{AttributeMap: map[string]*tracepb.AttributeValue{
@@ -264,7 +280,7 @@ func TestConsumer_Span(t *testing.T) {
 			td: consumerdata.TraceData{SourceFormat: "jaeger",
 				Node: &commonpb.Node{Identifier: &commonpb.ProcessIdentifier{HostName: "host-abc"}},
 				Spans: []*tracepb.Span{{
-					TraceId: []byte("FFx0"), SpanId: []byte("AAFF"), ParentSpanId: []byte("XXXX"),
+					TraceId: []byte{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 70, 70, 120, 48}, SpanId: []byte{0, 0, 0, 0, 65, 65, 70, 70}, ParentSpanId: []byte{0, 0, 0, 0, 88, 88, 88, 88},
 					StartTime: testStartTime(), EndTime: testEndTime(),
 					Name: testTruncatableString("HTTP GET"),
 					Attributes: &tracepb.Span_Attributes{AttributeMap: map[string]*tracepb.AttributeValue{
@@ -277,7 +293,7 @@ func TestConsumer_Span(t *testing.T) {
 			td: consumerdata.TraceData{SourceFormat: "jaeger",
 				Node: &commonpb.Node{Identifier: &commonpb.ProcessIdentifier{HostName: "host-abc"}},
 				Spans: []*tracepb.Span{{
-					ParentSpanId: []byte("abcd"), Kind: tracepb.Span_CLIENT,
+					ParentSpanId: []byte{0, 0, 0, 0, 97, 98, 99, 100}, Kind: tracepb.Span_CLIENT,
 					StartTime: testStartTime(), Attributes: &tracepb.Span_Attributes{AttributeMap: map[string]*tracepb.AttributeValue{
 						"db.statement": testAttributeStringValue("GET * from users"),
 						"db.instance":  testAttributeStringValue("db01"),
@@ -290,7 +306,7 @@ func TestConsumer_Span(t *testing.T) {
 			td: consumerdata.TraceData{SourceFormat: "jaeger",
 				Node: &commonpb.Node{Identifier: &commonpb.ProcessIdentifier{HostName: "host-abc"}},
 				Spans: []*tracepb.Span{{
-					ParentSpanId: []byte("abcd"), Kind: tracepb.Span_CLIENT,
+					ParentSpanId: []byte{0, 0, 0, 0, 97, 98, 99, 100}, Kind: tracepb.Span_CLIENT,
 					StartTime: testStartTime()}}}},
 	} {
 		t.Run(tc.name, func(t *testing.T) {

diff --git a/processor/otel/test_approved/metadata_jaeger_full-traceid.approved.json b/processor/otel/test_approved/metadata_jaeger_full-traceid.approved.json
@@ -0,0 +1,37 @@
+{
+    "events": [
+        {
+            "@timestamp": "2019-12-16T12:46:58.000768068Z",
+            "agent": {
+                "name": "Jaeger",
+                "version": "unknown"
+            },
+            "processor": {
+                "event": "transaction",
+                "name": "transaction"
+            },
+            "service": {
+                "language": {
+                    "name": "unknown"
+                },
+                "name": "unknown"
+            },
+            "timestamp": {
+                "us": 1576500418000768
+            },
+            "trace": {
+                "id": "464678300000000046467830"
+            },
+            "transaction": {
+                "duration": {
+                    "us": 0
+                },
+                "id": "41414646",
+                "name": "",
+                "result": "Success",
+                "sampled": true,
+                "type": "custom"
+            }
+        }
+    ]
+}
diff --git a/processor/otel/test_approved/metadata_minimal.approved.json b/processor/otel/test_approved/metadata_minimal.approved.json
@@ -20,13 +20,13 @@
                 "us": 1576500418000768
             },
             "trace": {
-                "id": "46467830"
+                "id": "00000000000000000000000046467830"
             },
             "transaction": {
                 "duration": {
                     "us": 0
                 },
-                "id": "41414646",
+                "id": "0000000041414646",
                 "name": "",
                 "result": "Success",
                 "sampled": true,

diff --git a/testdata/jaeger/batch_0.approved.json b/testdata/jaeger/batch_0.approved.json
@@ -37,7 +37,7 @@
                 "us": 1576827704953864
             },
             "trace": {
-                "id": "00000000000000007be2fd98d0973be3"
+                "id": "7be2fd98d0973be3"
             },
             "transaction": {
                 "duration": {
@@ -93,7 +93,7 @@
                 "us": 1576827705007552
             },
             "trace": {
-                "id": "00000000000000007be2fd98d0973be3"
+                "id": "7be2fd98d0973be3"
             },
             "transaction": {
                 "id": "7be2fd98d0973be3",
@@ -143,7 +143,7 @@
                 "us": 1576827705089431
             },
             "trace": {
-                "id": "00000000000000007be2fd98d0973be3"
+                "id": "7be2fd98d0973be3"
             },
             "transaction": {
                 "id": "7be2fd98d0973be3",
@@ -193,7 +193,7 @@
                 "us": 1576827705172530
             },
             "trace": {
-                "id": "00000000000000007be2fd98d0973be3"
+                "id": "7be2fd98d0973be3"
             },
             "transaction": {
                 "id": "7be2fd98d0973be3",