From 543b8961a480fb0d8926917f01ef60a01c8490b2 Mon Sep 17 00:00:00 2001 From: Andrew Wilkins Date: Wed, 10 Feb 2021 16:11:36 +0800 Subject: [PATCH 1/4] model: add scheme parameter to ParseURL If the main "URL" input does not contain a scheme, use the given scheme parameter. --- model/context.go | 9 ++++++--- model/error_test.go | 6 +++--- model/modeldecoder/rumv3/decoder.go | 2 +- model/modeldecoder/v2/decoder.go | 2 +- model/transaction_test.go | 6 +++--- 5 files changed, 14 insertions(+), 11 deletions(-) diff --git a/model/context.go b/model/context.go index ed4dd1ebb94..d8ad67ef900 100644 --- a/model/context.go +++ b/model/context.go @@ -57,17 +57,20 @@ type URL struct { Fragment *string } -func ParseURL(original, hostname string) *URL { +func ParseURL(original, defaultHostname, defaultScheme string) *URL { original = truncate(original) url, err := url.Parse(original) if err != nil { return &URL{Original: &original} } if url.Scheme == "" { - url.Scheme = "http" + url.Scheme = defaultScheme + if url.Scheme == "" { + url.Scheme = "http" + } } if url.Host == "" { - url.Host = hostname + url.Host = defaultHostname } full := truncate(url.String()) out := &URL{ diff --git a/model/error_test.go b/model/error_test.go index 3b9fa17cb8b..50d33b264fe 100644 --- a/model/error_test.go +++ b/model/error_test.go @@ -555,7 +555,7 @@ func TestErrorTransformPage(t *testing.T) { Error: Error{ ID: &id, Page: &Page{ - URL: ParseURL(urlExample, ""), + URL: ParseURL(urlExample, "", ""), Referer: nil, }, }, @@ -572,9 +572,9 @@ func TestErrorTransformPage(t *testing.T) { Error: Error{ ID: &id, Timestamp: time.Now(), - URL: ParseURL("https://localhost:8200/", ""), + URL: ParseURL("https://localhost:8200/", "", ""), Page: &Page{ - URL: ParseURL(urlExample, ""), + URL: ParseURL(urlExample, "", ""), Referer: nil, }, }, diff --git a/model/modeldecoder/rumv3/decoder.go b/model/modeldecoder/rumv3/decoder.go index 13e9ca45d2e..855f0c2c0a8 100644 --- a/model/modeldecoder/rumv3/decoder.go +++ b/model/modeldecoder/rumv3/decoder.go @@ -433,7 +433,7 @@ func mapToMetricsetModel(from *metricset, metadata *model.Metadata, reqTime time func mapToPageModel(from contextPage, out *model.Page) { if from.URL.IsSet() { - out.URL = model.ParseURL(from.URL.Val, "") + out.URL = model.ParseURL(from.URL.Val, "", "") } if from.Referer.IsSet() { referer := from.Referer.Val diff --git a/model/modeldecoder/v2/decoder.go b/model/modeldecoder/v2/decoder.go index acc90735556..35a60cec50e 100644 --- a/model/modeldecoder/v2/decoder.go +++ b/model/modeldecoder/v2/decoder.go @@ -578,7 +578,7 @@ func mapToMetricsetModel(from *metricset, metadata *model.Metadata, reqTime time func mapToPageModel(from contextPage, out *model.Page) { if from.URL.IsSet() { - out.URL = model.ParseURL(from.URL.Val, "") + out.URL = model.ParseURL(from.URL.Val, "", "") } if from.Referer.IsSet() { referer := from.Referer.Val diff --git a/model/transaction_test.go b/model/transaction_test.go index 563ca2b15f1..5fa32b34468 100644 --- a/model/transaction_test.go +++ b/model/transaction_test.go @@ -240,7 +240,7 @@ func TestTransactionTransformPage(t *testing.T) { Type: "tx", Duration: 65.98, Page: &Page{ - URL: ParseURL(urlExample, ""), + URL: ParseURL(urlExample, "", ""), Referer: nil, }, }, @@ -259,9 +259,9 @@ func TestTransactionTransformPage(t *testing.T) { Type: "tx", Timestamp: time.Now(), Duration: 65.98, - URL: ParseURL("https://localhost:8200/", ""), + URL: ParseURL("https://localhost:8200/", "", ""), Page: &Page{ - URL: ParseURL(urlExample, ""), + URL: ParseURL(urlExample, "", ""), Referer: nil, }, }, From 21d030aff33f098ea131a27c577915cdc3b38e37 Mon Sep 17 00:00:00 2001 From: Andrew Wilkins Date: Wed, 10 Feb 2021 16:13:08 +0800 Subject: [PATCH 2/4] processor/otel: more support for OTel conventions Extend our support for OpenTelemetry semantic conventions. --- processor/otel/builder.go | 121 +++++ processor/otel/consumer.go | 387 +++++++++++----- processor/otel/consumer_test.go | 425 ++++++++++++++++++ processor/otel/metadata.go | 17 +- .../jaeger_sampling_rate.approved.json | 8 +- .../metadata_jaeger-no-language.approved.json | 3 +- .../metadata_jaeger-version.approved.json | 3 +- .../metadata_jaeger.approved.json | 3 +- .../span_jaeger_custom.approved.json | 2 +- ...pan_jaeger_subtype_component.approved.json | 2 +- .../transaction_jaeger_custom.approved.json | 3 +- .../transaction_jaeger_full.approved.json | 2 +- ...action_jaeger_type_component.approved.json | 3 +- ...action_jaeger_type_messaging.approved.json | 3 +- 14 files changed, 843 insertions(+), 139 deletions(-) create mode 100644 processor/otel/builder.go diff --git a/processor/otel/builder.go b/processor/otel/builder.go new file mode 100644 index 00000000000..cafc7571d91 --- /dev/null +++ b/processor/otel/builder.go @@ -0,0 +1,121 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package otel + +import ( + "fmt" + + "github.com/elastic/apm-server/model" +) + +type transactionBuilder struct { + *model.Transaction + httpURL string + httpHost string + httpScheme string +} + +func (tx *transactionBuilder) setFramework(name, version string) { + if name == "" { + return + } + tx.Metadata.Service.Framework.Name = name + tx.Metadata.Service.Framework.Version = version +} + +func (tx *transactionBuilder) setHTTPMethod(method string) { + tx.ensureHTTPRequest() + tx.HTTP.Request.Method = method +} + +func (tx *transactionBuilder) setHTTPScheme(scheme string) { + tx.ensureHTTP() + tx.httpScheme = scheme +} + +func (tx *transactionBuilder) setHTTPURL(httpURL string) { + tx.ensureHTTP() + tx.httpURL = httpURL +} + +func (tx *transactionBuilder) setHTTPHost(hostport string) { + tx.ensureHTTP() + tx.httpHost = hostport +} + +func (tx transactionBuilder) setHTTPVersion(version string) { + tx.ensureHTTP() + tx.HTTP.Version = &version +} + +func (tx transactionBuilder) setHTTPRemoteAddr(remoteAddr string) { + tx.ensureHTTPRequest() + if tx.HTTP.Request.Socket == nil { + tx.HTTP.Request.Socket = &model.Socket{} + } + tx.HTTP.Request.Socket.RemoteAddress = &remoteAddr +} + +func (tx *transactionBuilder) setHTTPStatusCode(statusCode int) { + tx.ensureHTTP() + if tx.HTTP.Response == nil { + tx.HTTP.Response = &model.Resp{} + } + tx.HTTP.Response.MinimalResp.StatusCode = &statusCode + if tx.Outcome == outcomeUnknown { + if statusCode >= 500 { + tx.Outcome = outcomeFailure + } else { + tx.Outcome = outcomeSuccess + } + } + if tx.Result == "" { + tx.Result = httpStatusCodeResult(statusCode) + } +} + +func (tx *transactionBuilder) ensureHTTPRequest() { + tx.ensureHTTP() + if tx.HTTP.Request == nil { + tx.HTTP.Request = &model.Req{} + } +} + +func (tx *transactionBuilder) ensureHTTP() { + if tx.HTTP == nil { + tx.HTTP = &model.Http{} + } +} + +var standardStatusCodeResults = [...]string{ + "HTTP 1xx", + "HTTP 2xx", + "HTTP 3xx", + "HTTP 4xx", + "HTTP 5xx", +} + +// httpStatusCodeResult returns the transaction result value to use for the +// given HTTP status code. +func httpStatusCodeResult(statusCode int) string { + switch i := statusCode / 100; i { + case 1, 2, 3, 4, 5: + return standardStatusCodeResults[i-1] + } + return fmt.Sprintf("HTTP %d", statusCode) +} diff --git a/processor/otel/consumer.go b/processor/otel/consumer.go index 46fa9a607cd..a32a8c73c30 100644 --- a/processor/otel/consumer.go +++ b/processor/otel/consumer.go @@ -15,6 +15,23 @@ // specific language governing permissions and limitations // under the License. +// Portions copied from OpenTelemetry Collector (contrib), from the +// elastic exporter. +// +// Copyright 2020, OpenTelemetry Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package otel import ( @@ -26,6 +43,7 @@ import ( "strings" "go.opentelemetry.io/collector/consumer/pdata" + "go.opentelemetry.io/collector/translator/conventions" "github.com/elastic/beats/v7/libbeat/common" "github.com/elastic/beats/v7/libbeat/logp" @@ -36,8 +54,6 @@ import ( ) const ( - AgentNameJaeger = "Jaeger" - keywordLength = 1024 dot = "." underscore = "_" @@ -83,11 +99,16 @@ func (c *Consumer) convertResourceSpans(resourceSpans pdata.ResourceSpans, out * func (c *Consumer) convertInstrumentationLibrarySpans(in pdata.InstrumentationLibrarySpans, metadata model.Metadata, out *model.Batch) { otelSpans := in.Spans() for i := 0; i < otelSpans.Len(); i++ { - c.convertSpan(otelSpans.At(i), metadata, out) + c.convertSpan(otelSpans.At(i), in.InstrumentationLibrary(), metadata, out) } } -func (c *Consumer) convertSpan(otelSpan pdata.Span, metadata model.Metadata, out *model.Batch) { +func (c *Consumer) convertSpan( + otelSpan pdata.Span, + otelLibrary pdata.InstrumentationLibrary, + metadata model.Metadata, + out *model.Batch, +) { logger := logp.NewLogger(logs.Otel) root := !otelSpan.ParentSpanID().IsValid() @@ -120,8 +141,9 @@ func (c *Consumer) convertSpan(otelSpan pdata.Span, metadata model.Metadata, out Timestamp: startTime, Duration: durationMillis, Name: name, + Outcome: spanStatusOutcome(otelSpan.Status()), } - translateTransaction(otelSpan, metadata, transaction) + translateTransaction(otelSpan, otelLibrary, metadata, &transactionBuilder{Transaction: transaction}) out.Transactions = append(out.Transactions, transaction) } else { span = &model.Span{ @@ -132,7 +154,7 @@ func (c *Consumer) convertSpan(otelSpan pdata.Span, metadata model.Metadata, out Timestamp: startTime, Duration: durationMillis, Name: name, - Outcome: outcomeUnknown, + Outcome: spanStatusOutcome(otelSpan.Status()), } translateSpan(otelSpan, metadata, span) out.Spans = append(out.Spans, span) @@ -144,16 +166,27 @@ func (c *Consumer) convertSpan(otelSpan pdata.Span, metadata model.Metadata, out } } -func translateTransaction(span pdata.Span, metadata model.Metadata, event *model.Transaction) { +func translateTransaction( + span pdata.Span, + library pdata.InstrumentationLibrary, + metadata model.Metadata, + tx *transactionBuilder, +) { isJaeger := strings.HasPrefix(metadata.Service.Agent.Name, "Jaeger") labels := make(common.MapStr) - var http model.Http - var httpStatusCode int + + var ( + netHostName string + netHostPort int + netPeerIP string + netPeerPort int + ) + var message model.Message var component string - var outcome, result string - var isHTTP, isMessaging bool + var isMessaging bool var samplerType, samplerParam pdata.AttributeValue + var httpHostName string span.Attributes().ForEach(func(kDots string, v pdata.AttributeValue) { if isJaeger { switch kDots { @@ -174,43 +207,84 @@ func translateTransaction(span pdata.Span, metadata model.Metadata, event *model labels[k] = v.DoubleVal() case pdata.AttributeValueINT: switch kDots { - case "http.status_code": - httpStatusCode = int(v.IntVal()) - isHTTP = true + case conventions.AttributeHTTPStatusCode: + tx.setHTTPStatusCode(int(v.IntVal())) + case conventions.AttributeNetPeerPort: + netPeerPort = int(v.IntVal()) + case conventions.AttributeNetHostPort: + netHostPort = int(v.IntVal()) default: labels[k] = v.IntVal() } case pdata.AttributeValueSTRING: stringval := truncate(v.StringVal()) switch kDots { - case "span.kind": // filter out - case "http.method": - http.Request = &model.Req{Method: stringval} - isHTTP = true - case "http.url", "http.path": - event.URL = model.ParseURL(stringval, metadata.System.DetectedHostname) - isHTTP = true - case "http.status_code": + // http.* + case conventions.AttributeHTTPMethod: + tx.setHTTPMethod(stringval) + case conventions.AttributeHTTPURL, conventions.AttributeHTTPTarget, "http.path": + tx.setHTTPURL(stringval) + case conventions.AttributeHTTPHost: + tx.setHTTPHost(stringval) + case conventions.AttributeHTTPScheme: + tx.setHTTPScheme(stringval) + case conventions.AttributeHTTPStatusCode: if intv, err := strconv.Atoi(stringval); err == nil { - httpStatusCode = intv + tx.setHTTPStatusCode(intv) } - isHTTP = true case "http.protocol": - if strings.HasPrefix(stringval, "HTTP/") { - version := strings.TrimPrefix(stringval, "HTTP/") - http.Version = &version - } else { + if !strings.HasPrefix(stringval, "HTTP/") { + // Unexpected, store in labels for debugging. labels[k] = stringval + break + } + stringval = strings.TrimPrefix(stringval, "HTTP/") + fallthrough + case conventions.AttributeHTTPFlavor: + tx.setHTTPVersion(stringval) + case conventions.AttributeHTTPServerName: + httpHostName = stringval + case conventions.AttributeHTTPClientIP: + tx.Metadata.Client.IP = net.ParseIP(stringval) + case conventions.AttributeHTTPUserAgent: + tx.Metadata.UserAgent.Original = stringval + case "http.remote_addr": + // NOTE(axw) this is non-standard, sent by opentelemetry-go's othttp. + // It's semanticall equivalent to net.peer.ip+port. Standard attributes + // take precedence. + ip, port, err := net.SplitHostPort(stringval) + if err != nil { + ip = stringval } - isHTTP = true + if net.ParseIP(ip) != nil { + if netPeerIP == "" { + netPeerIP = ip + } + if netPeerPort == 0 { + netPeerPort, _ = strconv.Atoi(port) + } + } + + // net.* + case conventions.AttributeNetPeerIP: + netPeerIP = stringval + case conventions.AttributeNetHostName: + netHostName = stringval + + // messaging + // + // TODO(axw) translate OpenTelemtry messaging conventions. case "message_bus.destination": message.QueueName = &stringval isMessaging = true + + // miscellaneous + case "span.kind": // filter out case "type": - event.Type = stringval - case "service.version": - event.Metadata.Service.Version = stringval - case "component": + tx.Type = stringval + case conventions.AttributeServiceVersion: + tx.Metadata.Service.Version = stringval + case conventions.AttributeComponent: component = stringval fallthrough default: @@ -219,57 +293,81 @@ func translateTransaction(span pdata.Span, metadata model.Metadata, event *model } }) - if event.Type == "" { - if isHTTP { - event.Type = "request" + if tx.Type == "" { + if tx.HTTP != nil { + tx.Type = "request" } else if isMessaging { - event.Type = "messaging" + tx.Type = "messaging" } else if component != "" { - event.Type = component + tx.Type = component } else { - event.Type = "custom" + tx.Type = "custom" } } - if isHTTP { - if httpStatusCode > 0 { - http.Response = &model.Resp{MinimalResp: model.MinimalResp{StatusCode: &httpStatusCode}} - result = statusCodeResult(httpStatusCode) - outcome = serverStatusCodeOutcome(httpStatusCode) + if tx.HTTP != nil { + // Build the model.URL from tx.http{URL,Host,Scheme}. + httpHost := tx.httpHost + if httpHost == "" { + httpHost = httpHostName + if httpHost == "" { + httpHost = netHostName + if httpHost == "" { + httpHost = metadata.System.DetectedHostname + } + } + if httpHost != "" && netHostPort > 0 { + httpHost = net.JoinHostPort(httpHost, strconv.Itoa(netHostPort)) + } + } + tx.URL = model.ParseURL(tx.httpURL, httpHost, tx.httpScheme) + + // Set the remote address from net.peer.* + if tx.HTTP.Request != nil && netPeerIP != "" { + remoteAddr := netPeerIP + if netPeerPort > 0 { + remoteAddr = net.JoinHostPort(remoteAddr, strconv.Itoa(netPeerPort)) + } + tx.setHTTPRemoteAddr(remoteAddr) } - event.HTTP = &http - } else if isMessaging { - event.Message = &message } - if result == "" { - if span.Status().Code() == pdata.StatusCodeError { - result = "Error" - outcome = outcomeFailure - } else { - result = "Success" - outcome = outcomeSuccess - } + if isMessaging { + tx.Message = &message } - event.Result = result - event.Outcome = outcome if samplerType != (pdata.AttributeValue{}) { // The client has reported its sampling rate, so we can use it to extrapolate span metrics. - parseSamplerAttributes(samplerType, samplerParam, &event.RepresentativeCount, labels) + parseSamplerAttributes(samplerType, samplerParam, &tx.RepresentativeCount, labels) } - event.Labels = labels + if tx.Result == "" { + tx.Result = spanStatusResult(span.Status()) + } + tx.setFramework(library.Name(), library.Version()) + tx.Labels = labels } func translateSpan(span pdata.Span, metadata model.Metadata, event *model.Span) { isJaeger := strings.HasPrefix(metadata.Service.Agent.Name, "Jaeger") labels := make(common.MapStr) + var ( + netPeerName string + netPeerIP string + netPeerPort int + ) + + var ( + httpURL string + httpHost string + httpTarget string + httpScheme string = "http" + ) + var http model.HTTP var message model.Message var db model.DB - var destination model.Destination var destinationService model.DestinationService var isDBSpan, isHTTPSpan, isMessagingSpan bool var component string @@ -298,41 +396,56 @@ func translateSpan(span pdata.Span, metadata model.Metadata, event *model.Span) code := int(v.IntVal()) http.StatusCode = &code isHTTPSpan = true - case "peer.port": - port := int(v.IntVal()) - destination.Port = &port + case conventions.AttributeNetPeerPort, "peer.port": + netPeerPort = int(v.IntVal()) default: labels[k] = v.IntVal() } case pdata.AttributeValueSTRING: stringval := truncate(v.StringVal()) switch kDots { - case "span.kind": // filter out - case "http.url": - http.URL = &stringval + // http.* + case conventions.AttributeHTTPHost: + httpHost = stringval + isHTTPSpan = true + case conventions.AttributeHTTPScheme: + httpScheme = stringval + isHTTPSpan = true + case conventions.AttributeHTTPTarget: + httpTarget = stringval isHTTPSpan = true - case "http.method": + case conventions.AttributeHTTPURL: + httpURL = stringval + isHTTPSpan = true + case conventions.AttributeHTTPMethod: http.Method = &stringval isHTTPSpan = true + + // db.* case "sql.query": - db.Statement = &stringval if db.Type == nil { dbType := "sql" db.Type = &dbType } - isDBSpan = true - case "db.statement": + fallthrough + case conventions.AttributeDBStatement: db.Statement = &stringval isDBSpan = true - case "db.instance": + case conventions.AttributeDBName, "db.instance": db.Instance = &stringval isDBSpan = true - case "db.type": + case conventions.AttributeDBSystem, "db.type": db.Type = &stringval isDBSpan = true - case "db.user": + case conventions.AttributeDBUser: db.UserName = &stringval isDBSpan = true + + // net.* + case conventions.AttributeNetPeerName, "peer.hostname": + netPeerName = stringval + case conventions.AttributeNetPeerIP, "peer.ipv4", "peer.ipv6": + netPeerIP = stringval case "peer.address": destinationService.Resource = &stringval if !strings.ContainsRune(stringval, ':') || net.ParseIP(stringval) != nil { @@ -340,20 +453,25 @@ func translateSpan(span pdata.Span, metadata model.Metadata, event *model.Span) // or IP address; it could be something like // a JDBC connection string or ip:port. Ignore // values containing colons, except for IPv6. - destination.Address = &stringval + netPeerName = stringval } - case "peer.hostname", "peer.ipv4", "peer.ipv6": - destination.Address = &stringval - case "peer.service": + + // messaging + // + // TODO(axw) translate OpenTelemtry messaging conventions. + case "message_bus.destination": + message.QueueName = &stringval + isMessagingSpan = true + + // miscellaneous + case "span.kind": // filter out + case conventions.AttributePeerService: destinationService.Name = &stringval if destinationService.Resource == nil { // Prefer using peer.address for resource. destinationService.Resource = &stringval } - case "message_bus.destination": - message.QueueName = &stringval - isMessagingSpan = true - case "component": + case conventions.AttributeComponent: component = stringval fallthrough default: @@ -362,8 +480,34 @@ func translateSpan(span pdata.Span, metadata model.Metadata, event *model.Span) } }) - if http.URL != nil { - if fullURL, err := url.Parse(*http.URL); err == nil { + destPort := netPeerPort + destAddr := netPeerName + if destAddr == "" { + destAddr = netPeerIP + } + + if isHTTPSpan { + var fullURL *url.URL + if httpURL != "" { + fullURL, _ = url.Parse(httpURL) + } else if httpTarget != "" { + // Build http.url from http.scheme, http.target, etc. + if u, err := url.Parse(httpTarget); err == nil { + fullURL = u + fullURL.Scheme = httpScheme + if httpHost == "" { + // Set host from net.peer.* + httpHost = destAddr + if netPeerPort > 0 { + httpHost = net.JoinHostPort(httpHost, strconv.Itoa(destPort)) + } + } + fullURL.Host = httpHost + httpURL = fullURL.String() + } + } + if fullURL != nil { + http.URL = &httpURL url := url.URL{Scheme: fullURL.Scheme, Host: fullURL.Host} hostname := truncate(url.Hostname()) var port int @@ -376,13 +520,12 @@ func translateSpan(span pdata.Span, metadata model.Metadata, event *model.Span) // Set destination.{address,port} from the HTTP URL, // replacing peer.* based values to ensure consistency. - destination = model.Destination{Address: &hostname} + destAddr = hostname if port > 0 { - destination.Port = &port + destPort = port } - // Set destination.service.* from the HTTP URL, - // unless peer.service was specified. + // Set destination.service.* from the HTTP URL, unless peer.service was specified. if destinationService.Name == nil { resource := url.Host if port > 0 && port == schemeDefaultPort(url.Scheme) { @@ -402,14 +545,12 @@ func translateSpan(span pdata.Span, metadata model.Metadata, event *model.Span) } } - if destination != (model.Destination{}) { - event.Destination = &destination - } - switch { case isHTTPSpan: if http.StatusCode != nil { - event.Outcome = clientStatusCodeOutcome(*http.StatusCode) + if event.Outcome == outcomeUnknown { + event.Outcome = clientHTTPStatusCodeOutcome(*http.StatusCode) + } } event.Type = "external" subtype := "http" @@ -419,18 +560,30 @@ func translateSpan(span pdata.Span, metadata model.Metadata, event *model.Span) event.Type = "db" if db.Type != nil && *db.Type != "" { event.Subtype = db.Type + if destinationService.Name == nil { + // For database requests, we currently just identify + // the destination service by db.system. + destinationService.Name = event.Subtype + destinationService.Resource = event.Subtype + } } event.DB = &db case isMessagingSpan: event.Type = "messaging" event.Message = &message default: - event.Type = "custom" + event.Type = "app" if component != "" { event.Subtype = &component } } + if destAddr != "" { + event.Destination = &model.Destination{Address: &destAddr} + if destPort > 0 { + event.Destination.Port = &destPort + } + } if destinationService != (model.DestinationService{}) { if destinationService.Type == nil { // Copy span type to destination.service.type. @@ -479,6 +632,7 @@ func convertSpanEvent( isJaeger := strings.HasPrefix(metadata.Service.Agent.Name, "Jaeger") event.Attributes().ForEach(func(k string, v pdata.AttributeValue) { if !isJaeger { + // TODO(axw) translate OpenTelemetry exception span events. return } if v.Type() != pdata.AttributeValueSTRING { @@ -565,7 +719,7 @@ func addSpanCtxToErr(span *model.Span, hostname string, err *model.Error) { err.HTTP.Request = &model.Req{Method: *span.HTTP.Method} } if span.HTTP.URL != nil { - err.URL = model.ParseURL(*span.HTTP.URL, hostname) + err.URL = model.ParseURL(*span.HTTP.URL, hostname, "") } } } @@ -574,35 +728,34 @@ func replaceDots(s string) string { return strings.ReplaceAll(s, dot, underscore) } -// copied from elastic go-apm agent - -var standardStatusCodeResults = [...]string{ - "HTTP 1xx", - "HTTP 2xx", - "HTTP 3xx", - "HTTP 4xx", - "HTTP 5xx", -} - -// statusCodeResult returns the transaction result value to use for the given status code. -func statusCodeResult(statusCode int) string { - switch i := statusCode / 100; i { - case 1, 2, 3, 4, 5: - return standardStatusCodeResults[i-1] +// spanStatusOutcome returns the outcome for transactions and spans based on +// the given OTLP span status. +func spanStatusOutcome(status pdata.SpanStatus) string { + switch status.Code() { + case pdata.StatusCodeOk: + return outcomeSuccess + case pdata.StatusCodeError: + return outcomeFailure } - return fmt.Sprintf("HTTP %d", statusCode) + return outcomeUnknown } -// serverStatusCodeOutcome returns the transaction outcome value to use for the given status code. -func serverStatusCodeOutcome(statusCode int) string { - if statusCode >= 500 { - return outcomeFailure +// spanStatusResult returns the result for transactions based on the given +// OTLP span status. If the span status is unknown, an empty result string +// is returned. +func spanStatusResult(status pdata.SpanStatus) string { + switch status.Code() { + case pdata.StatusCodeOk: + return "Success" + case pdata.StatusCodeError: + return "Error" } - return outcomeSuccess + return "" } -// clientStatusCodeOutcome returns the span outcome value to use for the given status code. -func clientStatusCodeOutcome(statusCode int) string { +// clientHTTPStatusCodeOutcome returns the span outcome value to use for the +// given HTTP status code. +func clientHTTPStatusCodeOutcome(statusCode int) string { if statusCode >= 400 { return outcomeFailure } diff --git a/processor/otel/consumer_test.go b/processor/otel/consumer_test.go index d64b613c971..f4c26bc76a1 100644 --- a/processor/otel/consumer_test.go +++ b/processor/otel/consumer_test.go @@ -15,11 +15,29 @@ // specific language governing permissions and limitations // under the License. +// Portions copied from OpenTelemetry Collector (contrib), from the +// elastic exporter. +// +// Copyright 2020, OpenTelemetry Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package otel import ( "context" "fmt" + "net" "path/filepath" "testing" "time" @@ -31,6 +49,7 @@ import ( jaegertranslator "go.opentelemetry.io/collector/translator/trace/jaeger" "github.com/elastic/beats/v7/libbeat/beat" + "github.com/elastic/beats/v7/libbeat/common" "github.com/elastic/apm-server/approvaltest" "github.com/elastic/apm-server/beater/beatertest" @@ -50,6 +69,360 @@ func TestConsumer_ConsumeTraces_Empty(t *testing.T) { assert.NoError(t, consumer.ConsumeTraces(context.Background(), traces)) } +func TestHTTPTransactionURL(t *testing.T) { + test := func(t *testing.T, expected *model.URL, attrs map[string]pdata.AttributeValue) { + t.Helper() + tx := transformTransactionWithAttributes(t, attrs) + assert.Equal(t, expected, tx.URL) + } + + t.Run("scheme_host_target", func(t *testing.T) { + test(t, &model.URL{ + Scheme: newString("https"), + Original: newString("/foo?bar"), + Full: newString("https://testing.invalid:80/foo?bar"), + Path: newString("/foo"), + Query: newString("bar"), + Domain: newString("testing.invalid"), + Port: newInt(80), + }, map[string]pdata.AttributeValue{ + "http.scheme": pdata.NewAttributeValueString("https"), + "http.host": pdata.NewAttributeValueString("testing.invalid:80"), + "http.target": pdata.NewAttributeValueString("/foo?bar"), + }) + }) + t.Run("scheme_servername_nethostport_target", func(t *testing.T) { + test(t, &model.URL{ + Scheme: newString("https"), + Original: newString("/foo?bar"), + Full: newString("https://testing.invalid:80/foo?bar"), + Path: newString("/foo"), + Query: newString("bar"), + Domain: newString("testing.invalid"), + Port: newInt(80), + }, map[string]pdata.AttributeValue{ + "http.scheme": pdata.NewAttributeValueString("https"), + "http.server_name": pdata.NewAttributeValueString("testing.invalid"), + "net.host.port": pdata.NewAttributeValueInt(80), + "http.target": pdata.NewAttributeValueString("/foo?bar"), + }) + }) + t.Run("scheme_nethostname_nethostport_target", func(t *testing.T) { + test(t, &model.URL{ + Scheme: newString("https"), + Original: newString("/foo?bar"), + Full: newString("https://testing.invalid:80/foo?bar"), + Path: newString("/foo"), + Query: newString("bar"), + Domain: newString("testing.invalid"), + Port: newInt(80), + }, map[string]pdata.AttributeValue{ + "http.scheme": pdata.NewAttributeValueString("https"), + "net.host.name": pdata.NewAttributeValueString("testing.invalid"), + "net.host.port": pdata.NewAttributeValueInt(80), + "http.target": pdata.NewAttributeValueString("/foo?bar"), + }) + }) + t.Run("http.url", func(t *testing.T) { + test(t, &model.URL{ + Scheme: newString("https"), + Original: newString("https://testing.invalid:80/foo?bar"), + Full: newString("https://testing.invalid:80/foo?bar"), + Path: newString("/foo"), + Query: newString("bar"), + Domain: newString("testing.invalid"), + Port: newInt(80), + }, map[string]pdata.AttributeValue{ + "http.url": pdata.NewAttributeValueString("https://testing.invalid:80/foo?bar"), + }) + }) + t.Run("host_no_port", func(t *testing.T) { + test(t, &model.URL{ + Scheme: newString("https"), + Original: newString("/foo"), + Full: newString("https://testing.invalid/foo"), + Path: newString("/foo"), + Domain: newString("testing.invalid"), + }, map[string]pdata.AttributeValue{ + "http.scheme": pdata.NewAttributeValueString("https"), + "http.host": pdata.NewAttributeValueString("testing.invalid"), + "http.target": pdata.NewAttributeValueString("/foo"), + }) + }) + t.Run("ipv6_host_no_port", func(t *testing.T) { + test(t, &model.URL{ + Scheme: newString("https"), + Original: newString("/foo"), + Full: newString("https://[::1]/foo"), + Path: newString("/foo"), + Domain: newString("::1"), + }, map[string]pdata.AttributeValue{ + "http.scheme": pdata.NewAttributeValueString("https"), + "http.host": pdata.NewAttributeValueString("[::1]"), + "http.target": pdata.NewAttributeValueString("/foo"), + }) + }) + t.Run("default_scheme", func(t *testing.T) { + // scheme is set to "http" if it can't be deduced from attributes. + test(t, &model.URL{ + Scheme: newString("http"), + Original: newString("/foo"), + Full: newString("http://testing.invalid/foo"), + Path: newString("/foo"), + Domain: newString("testing.invalid"), + }, map[string]pdata.AttributeValue{ + "http.host": pdata.NewAttributeValueString("testing.invalid"), + "http.target": pdata.NewAttributeValueString("/foo"), + }) + }) +} + +func TestHTTPSpanURL(t *testing.T) { + test := func(t *testing.T, expected string, attrs map[string]pdata.AttributeValue) { + t.Helper() + span := transformSpanWithAttributes(t, attrs) + require.NotNil(t, span.HTTP) + require.NotNil(t, span.HTTP.URL) + assert.Equal(t, expected, *span.HTTP.URL) + } + + t.Run("host.url", func(t *testing.T) { + test(t, "https://testing.invalid:80/foo?bar", map[string]pdata.AttributeValue{ + "http.url": pdata.NewAttributeValueString("https://testing.invalid:80/foo?bar"), + }) + }) + t.Run("scheme_host_target", func(t *testing.T) { + test(t, "https://testing.invalid:80/foo?bar", map[string]pdata.AttributeValue{ + "http.scheme": pdata.NewAttributeValueString("https"), + "http.host": pdata.NewAttributeValueString("testing.invalid:80"), + "http.target": pdata.NewAttributeValueString("/foo?bar"), + }) + }) + t.Run("scheme_netpeername_netpeerport_target", func(t *testing.T) { + test(t, "https://testing.invalid:80/foo?bar", map[string]pdata.AttributeValue{ + "http.scheme": pdata.NewAttributeValueString("https"), + "net.peer.name": pdata.NewAttributeValueString("testing.invalid"), + "net.peer.ip": pdata.NewAttributeValueString("::1"), // net.peer.name preferred + "net.peer.port": pdata.NewAttributeValueInt(80), + "http.target": pdata.NewAttributeValueString("/foo?bar"), + }) + }) + t.Run("scheme_netpeerip_netpeerport_target", func(t *testing.T) { + test(t, "https://[::1]:80/foo?bar", map[string]pdata.AttributeValue{ + "http.scheme": pdata.NewAttributeValueString("https"), + "net.peer.ip": pdata.NewAttributeValueString("::1"), + "net.peer.port": pdata.NewAttributeValueInt(80), + "http.target": pdata.NewAttributeValueString("/foo?bar"), + }) + }) + t.Run("default_scheme", func(t *testing.T) { + // scheme is set to "http" if it can't be deduced from attributes. + test(t, "http://testing.invalid/foo", map[string]pdata.AttributeValue{ + "http.host": pdata.NewAttributeValueString("testing.invalid"), + "http.target": pdata.NewAttributeValueString("/foo"), + }) + }) +} + +func TestHTTPSpanDestination(t *testing.T) { + test := func(t *testing.T, expectedDestination *model.Destination, expectedDestinationService *model.DestinationService, attrs map[string]pdata.AttributeValue) { + t.Helper() + span := transformSpanWithAttributes(t, attrs) + assert.Equal(t, expectedDestination, span.Destination) + assert.Equal(t, expectedDestinationService, span.DestinationService) + } + + t.Run("url_default_port_specified", func(t *testing.T) { + test(t, &model.Destination{ + Address: newString("testing.invalid"), + Port: newInt(443), + }, &model.DestinationService{ + Type: newString("external"), + Name: newString("https://testing.invalid"), + Resource: newString("testing.invalid:443"), + }, map[string]pdata.AttributeValue{ + "http.url": pdata.NewAttributeValueString("https://testing.invalid:443/foo?bar"), + }) + }) + t.Run("url_port_scheme", func(t *testing.T) { + test(t, &model.Destination{ + Address: newString("testing.invalid"), + Port: newInt(443), + }, &model.DestinationService{ + Type: newString("external"), + Name: newString("https://testing.invalid"), + Resource: newString("testing.invalid:443"), + }, map[string]pdata.AttributeValue{ + "http.url": pdata.NewAttributeValueString("https://testing.invalid/foo?bar"), + }) + }) + t.Run("url_non_default_port", func(t *testing.T) { + test(t, &model.Destination{ + Address: newString("testing.invalid"), + Port: newInt(444), + }, &model.DestinationService{ + Type: newString("external"), + Name: newString("https://testing.invalid:444"), + Resource: newString("testing.invalid:444"), + }, map[string]pdata.AttributeValue{ + "http.url": pdata.NewAttributeValueString("https://testing.invalid:444/foo?bar"), + }) + }) + t.Run("scheme_host_target", func(t *testing.T) { + test(t, &model.Destination{ + Address: newString("testing.invalid"), + Port: newInt(444), + }, &model.DestinationService{ + Type: newString("external"), + Name: newString("https://testing.invalid:444"), + Resource: newString("testing.invalid:444"), + }, map[string]pdata.AttributeValue{ + "http.scheme": pdata.NewAttributeValueString("https"), + "http.host": pdata.NewAttributeValueString("testing.invalid:444"), + "http.target": pdata.NewAttributeValueString("/foo?bar"), + }) + }) + t.Run("scheme_netpeername_nethostport_target", func(t *testing.T) { + test(t, &model.Destination{ + Address: newString("::1"), + Port: newInt(444), + }, &model.DestinationService{ + Type: newString("external"), + Name: newString("https://[::1]:444"), + Resource: newString("[::1]:444"), + }, map[string]pdata.AttributeValue{ + "http.scheme": pdata.NewAttributeValueString("https"), + "net.peer.ip": pdata.NewAttributeValueString("::1"), + "net.peer.port": pdata.NewAttributeValueInt(444), + "http.target": pdata.NewAttributeValueString("/foo?bar"), + }) + }) +} + +func TestHTTPTransactionRequestSocketRemoteAddr(t *testing.T) { + test := func(t *testing.T, expected string, attrs map[string]pdata.AttributeValue) { + // "http.method" is a required attribute for HTTP spans, + // and its presence causes the transaction's HTTP request + // context to be built. + attrs["http.method"] = pdata.NewAttributeValueString("POST") + + tx := transformTransactionWithAttributes(t, attrs) + require.NotNil(t, tx.HTTP) + require.NotNil(t, tx.HTTP.Request) + require.NotNil(t, tx.HTTP.Request.Socket) + assert.Equal(t, &expected, tx.HTTP.Request.Socket.RemoteAddress) + } + + t.Run("net.peer.ip_port", func(t *testing.T) { + test(t, "192.168.0.1:1234", map[string]pdata.AttributeValue{ + "net.peer.ip": pdata.NewAttributeValueString("192.168.0.1"), + "net.peer.port": pdata.NewAttributeValueInt(1234), + }) + }) + t.Run("net.peer.ip", func(t *testing.T) { + test(t, "192.168.0.1", map[string]pdata.AttributeValue{ + "net.peer.ip": pdata.NewAttributeValueString("192.168.0.1"), + }) + }) + t.Run("http.remote_addr", func(t *testing.T) { + test(t, "192.168.0.1:1234", map[string]pdata.AttributeValue{ + "http.remote_addr": pdata.NewAttributeValueString("192.168.0.1:1234"), + }) + }) + t.Run("http.remote_addr_no_port", func(t *testing.T) { + test(t, "192.168.0.1", map[string]pdata.AttributeValue{ + "http.remote_addr": pdata.NewAttributeValueString("192.168.0.1"), + }) + }) +} + +func TestHTTPTransactionFlavor(t *testing.T) { + tx := transformTransactionWithAttributes(t, map[string]pdata.AttributeValue{ + "http.flavor": pdata.NewAttributeValueString("1.1"), + }) + assert.Equal(t, newString("1.1"), tx.HTTP.Version) +} + +func TestHTTPTransactionUserAgent(t *testing.T) { + tx := transformTransactionWithAttributes(t, map[string]pdata.AttributeValue{ + "http.user_agent": pdata.NewAttributeValueString("Foo/bar (baz)"), + }) + assert.Equal(t, model.UserAgent{Original: "Foo/bar (baz)"}, tx.Metadata.UserAgent) +} + +func TestHTTPTransactionClientIP(t *testing.T) { + tx := transformTransactionWithAttributes(t, map[string]pdata.AttributeValue{ + "http.client_ip": pdata.NewAttributeValueString("256.257.258.259"), + }) + assert.Equal(t, net.ParseIP("256.257.258.259"), tx.Metadata.Client.IP) +} + +func TestHTTPTransactionStatusCode(t *testing.T) { + tx := transformTransactionWithAttributes(t, map[string]pdata.AttributeValue{ + "http.status_code": pdata.NewAttributeValueInt(200), + }) + assert.Equal(t, newInt(200), tx.HTTP.Response.StatusCode) +} + +func TestDatabaseSpan(t *testing.T) { + // https://github.com/open-telemetry/opentelemetry-specification/blob/master/specification/trace/semantic_conventions/database.md#mysql + connectionString := "Server=shopdb.example.com;Database=ShopDb;Uid=billing_user;TableCache=true;UseCompression=True;MinimumPoolSize=10;MaximumPoolSize=50;" + span := transformSpanWithAttributes(t, map[string]pdata.AttributeValue{ + "db.system": pdata.NewAttributeValueString("mysql"), + "db.connection_string": pdata.NewAttributeValueString(connectionString), + "db.user": pdata.NewAttributeValueString("billing_user"), + "db.name": pdata.NewAttributeValueString("ShopDb"), + "db.statement": pdata.NewAttributeValueString("SELECT * FROM orders WHERE order_id = 'o4711'"), + "net.peer.name": pdata.NewAttributeValueString("shopdb.example.com"), + "net.peer.ip": pdata.NewAttributeValueString("192.0.2.12"), + "net.peer.port": pdata.NewAttributeValueInt(3306), + "net.transport": pdata.NewAttributeValueString("IP.TCP"), + }) + + assert.Equal(t, "db", span.Type) + assert.Equal(t, newString("mysql"), span.Subtype) + assert.Nil(t, span.Action) + + assert.Equal(t, &model.DB{ + Instance: newString("ShopDb"), + Statement: newString("SELECT * FROM orders WHERE order_id = 'o4711'"), + Type: newString("mysql"), + UserName: newString("billing_user"), + }, span.DB) + + assert.Equal(t, common.MapStr{ + "db_connection_string": connectionString, + "net_transport": "IP.TCP", + }, span.Labels) + + assert.Equal(t, &model.Destination{ + Address: newString("shopdb.example.com"), + Port: newInt(3306), + }, span.Destination) + + assert.Equal(t, &model.DestinationService{ + Type: newString("db"), + Name: newString("mysql"), + Resource: newString("mysql"), + }, span.DestinationService) +} + +func TestInstrumentationLibrary(t *testing.T) { + traces, spans := newTracesSpans() + spans.InstrumentationLibrary().SetName("library-name") + spans.InstrumentationLibrary().SetVersion("1.2.3") + otelSpan := pdata.NewSpan() + otelSpan.SetTraceID(pdata.NewTraceID([16]byte{1})) + otelSpan.SetSpanID(pdata.NewSpanID([8]byte{2})) + spans.Spans().Append(otelSpan) + events := transformTraces(t, traces) + require.Len(t, events, 1) + tx := events[0].(*model.Transaction) + + assert.Equal(t, "library-name", tx.Metadata.Service.Framework.Name) + assert.Equal(t, "1.2.3", tx.Metadata.Service.Framework.Version) +} + func TestConsumer_JaegerMetadata(t *testing.T) { jaegerBatch := jaegermodel.Batch{ Spans: []*jaegermodel.Span{{ @@ -538,3 +911,55 @@ func jaegerKeyValue(k string, v interface{}) jaegermodel.KeyValue { } return kv } + +func transformTransactionWithAttributes(t *testing.T, attrs map[string]pdata.AttributeValue) *model.Transaction { + traces, spans := newTracesSpans() + otelSpan := pdata.NewSpan() + otelSpan.SetTraceID(pdata.NewTraceID([16]byte{1})) + otelSpan.SetSpanID(pdata.NewSpanID([8]byte{2})) + otelSpan.Attributes().InitFromMap(attrs) + spans.Spans().Append(otelSpan) + events := transformTraces(t, traces) + require.Len(t, events, 1) + return events[0].(*model.Transaction) +} + +func transformSpanWithAttributes(t *testing.T, attrs map[string]pdata.AttributeValue) *model.Span { + traces, spans := newTracesSpans() + otelSpan := pdata.NewSpan() + otelSpan.SetTraceID(pdata.NewTraceID([16]byte{1})) + otelSpan.SetSpanID(pdata.NewSpanID([8]byte{2})) + otelSpan.SetParentSpanID(pdata.NewSpanID([8]byte{3})) + otelSpan.Attributes().InitFromMap(attrs) + spans.Spans().Append(otelSpan) + events := transformTraces(t, traces) + require.Len(t, events, 1) + return events[0].(*model.Span) +} + +func transformTraces(t *testing.T, traces pdata.Traces) []transform.Transformable { + var events []transform.Transformable + reporter := func(ctx context.Context, req publish.PendingReq) error { + events = append(events, req.Transformables...) + return nil + } + require.NoError(t, (&Consumer{Reporter: reporter}).ConsumeTraces(context.Background(), traces)) + return events +} + +func newTracesSpans() (pdata.Traces, pdata.InstrumentationLibrarySpans) { + traces := pdata.NewTraces() + resourceSpans := pdata.NewResourceSpans() + librarySpans := pdata.NewInstrumentationLibrarySpans() + resourceSpans.InstrumentationLibrarySpans().Append(librarySpans) + traces.ResourceSpans().Append(resourceSpans) + return traces, librarySpans +} + +func newString(s string) *string { + return &s +} + +func newInt(v int) *int { + return &v +} diff --git a/processor/otel/metadata.go b/processor/otel/metadata.go index 2949e178bf1..8eb506b6588 100644 --- a/processor/otel/metadata.go +++ b/processor/otel/metadata.go @@ -20,6 +20,7 @@ package otel import ( "fmt" "net" + "regexp" "strings" "go.opentelemetry.io/collector/consumer/pdata" @@ -29,12 +30,20 @@ import ( "github.com/elastic/beats/v7/libbeat/common" ) +const ( + AgentNameJaeger = "Jaeger" +) + +var ( + serviceNameInvalidRegexp = regexp.MustCompile("[^a-zA-Z0-9 _-]") +) + func translateResourceMetadata(resource pdata.Resource, out *model.Metadata) { var exporterVersion string resource.Attributes().ForEach(func(k string, v pdata.AttributeValue) { switch k { case conventions.AttributeServiceName: - out.Service.Name = truncate(v.StringVal()) + out.Service.Name = cleanServiceName(v.StringVal()) case conventions.AttributeServiceVersion: out.Service.Version = truncate(v.StringVal()) case conventions.AttributeServiceInstance: @@ -83,7 +92,7 @@ func translateResourceMetadata(resource pdata.Resource, out *model.Metadata) { if v := versionParts[len(versionParts)-1]; v != "" { out.Service.Agent.Version = v } - out.Service.Agent.Name = "Jaeger" + out.Service.Agent.Name = AgentNameJaeger // Translate known Jaeger labels. if clientUUID, ok := out.Labels["client-uuid"].(string); ok { @@ -115,6 +124,10 @@ func translateResourceMetadata(resource pdata.Resource, out *model.Metadata) { } } +func cleanServiceName(name string) string { + return serviceNameInvalidRegexp.ReplaceAllString(truncate(name), "_") +} + func ifaceAttributeValue(v pdata.AttributeValue) interface{} { switch v.Type() { case pdata.AttributeValueSTRING: diff --git a/processor/otel/test_approved/jaeger_sampling_rate.approved.json b/processor/otel/test_approved/jaeger_sampling_rate.approved.json index ba3c06f4fa0..28bfb67a1b4 100644 --- a/processor/otel/test_approved/jaeger_sampling_rate.approved.json +++ b/processor/otel/test_approved/jaeger_sampling_rate.approved.json @@ -9,7 +9,7 @@ "data_stream.dataset": "apm", "data_stream.type": "traces", "event": { - "outcome": "success" + "outcome": "unknown" }, "host": { "hostname": "host-abc", @@ -36,7 +36,6 @@ "us": 79000000 }, "id": "", - "result": "Success", "sampled": true, "type": "custom" } @@ -50,7 +49,7 @@ "data_stream.dataset": "apm", "data_stream.type": "traces", "event": { - "outcome": "success" + "outcome": "unknown" }, "host": { "hostname": "host-abc", @@ -81,7 +80,6 @@ "us": 79000000 }, "id": "", - "result": "Success", "sampled": true, "type": "custom" } @@ -122,7 +120,7 @@ "us": 79000000 }, "name": "", - "type": "custom" + "type": "app" }, "timestamp": { "us": 1576500418000768 diff --git a/processor/otel/test_approved/metadata_jaeger-no-language.approved.json b/processor/otel/test_approved/metadata_jaeger-no-language.approved.json index 966a2ffbbc3..0dc34aa4121 100644 --- a/processor/otel/test_approved/metadata_jaeger-no-language.approved.json +++ b/processor/otel/test_approved/metadata_jaeger-no-language.approved.json @@ -9,7 +9,7 @@ "data_stream.dataset": "apm", "data_stream.type": "traces", "event": { - "outcome": "success" + "outcome": "unknown" }, "processor": { "event": "transaction", @@ -32,7 +32,6 @@ "us": 0 }, "id": "0000000041414646", - "result": "Success", "sampled": true, "type": "custom" } diff --git a/processor/otel/test_approved/metadata_jaeger-version.approved.json b/processor/otel/test_approved/metadata_jaeger-version.approved.json index f04dd014c9e..a648c2fad35 100644 --- a/processor/otel/test_approved/metadata_jaeger-version.approved.json +++ b/processor/otel/test_approved/metadata_jaeger-version.approved.json @@ -9,7 +9,7 @@ "data_stream.dataset": "apm", "data_stream.type": "traces", "event": { - "outcome": "success" + "outcome": "unknown" }, "processor": { "event": "transaction", @@ -32,7 +32,6 @@ "us": 0 }, "id": "0000000041414646", - "result": "Success", "sampled": true, "type": "custom" } diff --git a/processor/otel/test_approved/metadata_jaeger.approved.json b/processor/otel/test_approved/metadata_jaeger.approved.json index 2e288ea1ab2..1a9a8aa2057 100644 --- a/processor/otel/test_approved/metadata_jaeger.approved.json +++ b/processor/otel/test_approved/metadata_jaeger.approved.json @@ -10,7 +10,7 @@ "data_stream.dataset": "apm", "data_stream.type": "traces", "event": { - "outcome": "success" + "outcome": "unknown" }, "host": { "hostname": "host-foo", @@ -45,7 +45,6 @@ "us": 0 }, "id": "0000000041414646", - "result": "Success", "sampled": true, "type": "custom" } diff --git a/processor/otel/test_approved/span_jaeger_custom.approved.json b/processor/otel/test_approved/span_jaeger_custom.approved.json index 5288a2b88ca..5ceab2067b1 100644 --- a/processor/otel/test_approved/span_jaeger_custom.approved.json +++ b/processor/otel/test_approved/span_jaeger_custom.approved.json @@ -37,7 +37,7 @@ }, "id": "0000000041414646", "name": "", - "type": "custom" + "type": "app" }, "timestamp": { "us": 1576500418000768 diff --git a/processor/otel/test_approved/span_jaeger_subtype_component.approved.json b/processor/otel/test_approved/span_jaeger_subtype_component.approved.json index 48bd6ecef37..bab6c03e440 100644 --- a/processor/otel/test_approved/span_jaeger_subtype_component.approved.json +++ b/processor/otel/test_approved/span_jaeger_subtype_component.approved.json @@ -41,7 +41,7 @@ "id": "0000000041414646", "name": "", "subtype": "whatever", - "type": "custom" + "type": "app" }, "timestamp": { "us": 1576500418000768 diff --git a/processor/otel/test_approved/transaction_jaeger_custom.approved.json b/processor/otel/test_approved/transaction_jaeger_custom.approved.json index 3b531443a3a..5e82aabde88 100644 --- a/processor/otel/test_approved/transaction_jaeger_custom.approved.json +++ b/processor/otel/test_approved/transaction_jaeger_custom.approved.json @@ -9,7 +9,7 @@ "data_stream.dataset": "apm", "data_stream.type": "traces", "event": { - "outcome": "success" + "outcome": "unknown" }, "host": { "hostname": "host-abc", @@ -36,7 +36,6 @@ "us": 0 }, "id": "", - "result": "Success", "sampled": true, "type": "custom" } diff --git a/processor/otel/test_approved/transaction_jaeger_full.approved.json b/processor/otel/test_approved/transaction_jaeger_full.approved.json index 8fb1dfdac67..a7299ac5dea 100644 --- a/processor/otel/test_approved/transaction_jaeger_full.approved.json +++ b/processor/otel/test_approved/transaction_jaeger_full.approved.json @@ -9,7 +9,7 @@ "data_stream.dataset": "apm", "data_stream.type": "traces", "event": { - "outcome": "success" + "outcome": "failure" }, "host": { "hostname": "host-abc", diff --git a/processor/otel/test_approved/transaction_jaeger_type_component.approved.json b/processor/otel/test_approved/transaction_jaeger_type_component.approved.json index 9ae85c5d296..def3ffc0f73 100644 --- a/processor/otel/test_approved/transaction_jaeger_type_component.approved.json +++ b/processor/otel/test_approved/transaction_jaeger_type_component.approved.json @@ -9,7 +9,7 @@ "data_stream.dataset": "apm", "data_stream.type": "traces", "event": { - "outcome": "success" + "outcome": "unknown" }, "host": { "hostname": "host-abc", @@ -36,7 +36,6 @@ "us": 0 }, "id": "", - "result": "Success", "sampled": true, "type": "amqp" } diff --git a/processor/otel/test_approved/transaction_jaeger_type_messaging.approved.json b/processor/otel/test_approved/transaction_jaeger_type_messaging.approved.json index 1fe49b9bcd4..5867df8200e 100644 --- a/processor/otel/test_approved/transaction_jaeger_type_messaging.approved.json +++ b/processor/otel/test_approved/transaction_jaeger_type_messaging.approved.json @@ -9,7 +9,7 @@ "data_stream.dataset": "apm", "data_stream.type": "traces", "event": { - "outcome": "success" + "outcome": "unknown" }, "host": { "hostname": "host-abc", @@ -44,7 +44,6 @@ "name": "queue-abc" } }, - "result": "Success", "sampled": true, "type": "messaging" } From 78207c37a9a109dabe45c8458809022cb86c9e7b Mon Sep 17 00:00:00 2001 From: Andrew Wilkins Date: Wed, 10 Feb 2021 16:30:46 +0800 Subject: [PATCH 3/4] Changelog --- changelogs/head.asciidoc | 2 ++ 1 file changed, 2 insertions(+) diff --git a/changelogs/head.asciidoc b/changelogs/head.asciidoc index 0163da75866..951cb42cc5f 100644 --- a/changelogs/head.asciidoc +++ b/changelogs/head.asciidoc @@ -6,6 +6,8 @@ https://github.com/elastic/apm-server/compare/7.11\...master[View commits] [float] ==== Breaking Changes * Leading 0s are no longer removed from trace/span ids if they are created by Jaeger {pull}4671[4671] +* Jaeger spans will now have a type of "app" where they previously were "custom" {pull}4711[4711] +* Jaeger spans may now have a (more accurate) outcome of "unknown" where they previously were "success" {pull}4711[4711] [float] ==== Bug fixes From 5fb860f0ab4151cc7e6686c9e8562116e462c63e Mon Sep 17 00:00:00 2001 From: Andrew Wilkins Date: Wed, 10 Feb 2021 18:47:07 +0800 Subject: [PATCH 4/4] Fix approvals --- .../approvals/TestOTLPGRPC.approved.json | 3 ++ testdata/jaeger/batch_0.approved.json | 3 +- testdata/jaeger/batch_1.approved.json | 34 +++++++++---------- ...ger_batch_0_auth_tag_removed.approved.json | 1 - ...jaeger_batch_0_authorization.approved.json | 1 - tests/system/jaeger_span.approved.json | 1 - 6 files changed, 21 insertions(+), 22 deletions(-) diff --git a/systemtest/approvals/TestOTLPGRPC.approved.json b/systemtest/approvals/TestOTLPGRPC.approved.json index b6a6490b5a0..ffc13186fc1 100644 --- a/systemtest/approvals/TestOTLPGRPC.approved.json +++ b/systemtest/approvals/TestOTLPGRPC.approved.json @@ -26,6 +26,9 @@ "name": "transaction" }, "service": { + "framework": { + "name": "systemtest" + }, "language": { "name": "unknown" }, diff --git a/testdata/jaeger/batch_0.approved.json b/testdata/jaeger/batch_0.approved.json index 45221240edf..cf4421d6d24 100644 --- a/testdata/jaeger/batch_0.approved.json +++ b/testdata/jaeger/batch_0.approved.json @@ -8,7 +8,7 @@ "version": "2.20.1" }, "event": { - "outcome": "success" + "outcome": "unknown" }, "host": { "hostname": "host01", @@ -48,7 +48,6 @@ }, "id": "7be2fd98d0973be3", "name": "Driver::findNearest", - "result": "Success", "sampled": true, "type": "custom" } diff --git a/testdata/jaeger/batch_1.approved.json b/testdata/jaeger/batch_1.approved.json index b9926744680..9feec9e765d 100644 --- a/testdata/jaeger/batch_1.approved.json +++ b/testdata/jaeger/batch_1.approved.json @@ -40,7 +40,7 @@ }, "id": "6e09e8bcefd6b828", "name": "FindDriverIDs", - "type": "custom" + "type": "app" }, "timestamp": { "us": 1576827704954062 @@ -57,7 +57,7 @@ "version": "2.20.1" }, "event": { - "outcome": "unknown" + "outcome": "failure" }, "host": { "hostname": "host01", @@ -89,7 +89,7 @@ }, "id": "333295bfb438ea03", "name": "GetDriver", - "type": "custom" + "type": "app" }, "timestamp": { "us": 1576827704973809 @@ -138,7 +138,7 @@ }, "id": "627c37a97e475c2f", "name": "GetDriver", - "type": "custom" + "type": "app" }, "timestamp": { "us": 1576827705007578 @@ -187,7 +187,7 @@ }, "id": "7bd7663d39c5a847", "name": "GetDriver", - "type": "custom" + "type": "app" }, "timestamp": { "us": 1576827705016845 @@ -236,7 +236,7 @@ }, "id": "6b4051dd2a5e2366", "name": "GetDriver", - "type": "custom" + "type": "app" }, "timestamp": { "us": 1576827705029415 @@ -285,7 +285,7 @@ }, "id": "6df97a86b9b3451b", "name": "GetDriver", - "type": "custom" + "type": "app" }, "timestamp": { "us": 1576827705040082 @@ -302,7 +302,7 @@ "version": "2.20.1" }, "event": { - "outcome": "unknown" + "outcome": "failure" }, "host": { "hostname": "host01", @@ -334,7 +334,7 @@ }, "id": "614811d6c498bfb0", "name": "GetDriver", - "type": "custom" + "type": "app" }, "timestamp": { "us": 1576827705054046 @@ -383,7 +383,7 @@ }, "id": "231604559da84d61", "name": "GetDriver", - "type": "custom" + "type": "app" }, "timestamp": { "us": 1576827705089459 @@ -432,7 +432,7 @@ }, "id": "61f7ecf24d13c36a", "name": "GetDriver", - "type": "custom" + "type": "app" }, "timestamp": { "us": 1576827705101278 @@ -481,7 +481,7 @@ }, "id": "2ef335bad24accc2", "name": "GetDriver", - "type": "custom" + "type": "app" }, "timestamp": { "us": 1576827705113531 @@ -530,7 +530,7 @@ }, "id": "38ec645e7201224d", "name": "GetDriver", - "type": "custom" + "type": "app" }, "timestamp": { "us": 1576827705125567 @@ -547,7 +547,7 @@ "version": "2.20.1" }, "event": { - "outcome": "unknown" + "outcome": "failure" }, "host": { "hostname": "host01", @@ -579,7 +579,7 @@ }, "id": "0242ee3774d9eab1", "name": "GetDriver", - "type": "custom" + "type": "app" }, "timestamp": { "us": 1576827705132896 @@ -628,7 +628,7 @@ }, "id": "6a63d1e81cfc7d95", "name": "GetDriver", - "type": "custom" + "type": "app" }, "timestamp": { "us": 1576827705172618 @@ -677,7 +677,7 @@ }, "id": "2b4c28f02b272f17", "name": "GetDriver", - "type": "custom" + "type": "app" }, "timestamp": { "us": 1576827705186670 diff --git a/tests/system/jaeger_batch_0_auth_tag_removed.approved.json b/tests/system/jaeger_batch_0_auth_tag_removed.approved.json index 44c08a2204e..7ea1b33323d 100644 --- a/tests/system/jaeger_batch_0_auth_tag_removed.approved.json +++ b/tests/system/jaeger_batch_0_auth_tag_removed.approved.json @@ -59,7 +59,6 @@ }, "id": "7be2fd98d0973be3", "name": "Driver::findNearest", - "result": "Success", "sampled": true, "type": "custom" } diff --git a/tests/system/jaeger_batch_0_authorization.approved.json b/tests/system/jaeger_batch_0_authorization.approved.json index 781bc75aeb6..d7bf815c2b1 100644 --- a/tests/system/jaeger_batch_0_authorization.approved.json +++ b/tests/system/jaeger_batch_0_authorization.approved.json @@ -59,7 +59,6 @@ }, "id": "7be2fd98d0973be3", "name": "Driver::findNearest", - "result": "Success", "sampled": true, "type": "custom" } diff --git a/tests/system/jaeger_span.approved.json b/tests/system/jaeger_span.approved.json index d1bbb9aaa8e..ac21b5ee496 100644 --- a/tests/system/jaeger_span.approved.json +++ b/tests/system/jaeger_span.approved.json @@ -55,7 +55,6 @@ }, "id": "5025e08c7fef6542", "name": "test_span", - "result": "Success", "sampled": true, "type": "custom" }