Add support for OTel Java runtime metrics (#19)

* Add java mappings * Debug logs * More Debug logs * Change to use md type * Change to non_heap and remove metricType * Add unit test * Refactor runtime metric mappings * Change runtime metric attribute definition * Handle metrics with multiple attributes, add code to remove mapping attributes * Move runtime metric mappings to separate file * Generate licenses * Support mapping histograms, add rest of jvm mappings * Generate licenses * License patch * Fix lint * Add type for runtimeMetricMappingList * Debug logs * Debug logs * Use Str instead of AsString * Change attribute values * Remove debug logs * Remove break in matchesAttributes section to map all DataPoints
DataDog · Apr 12, 2023 · 8781a60 · 8781a60
1 parent 412f048
commit 8781a60
Show file tree

Hide file tree

Showing 6 changed files with 521 additions and 94 deletions.
diff --git a/LICENSE-3rdparty.csv b/LICENSE-3rdparty.csv
@@ -203,6 +203,8 @@ pkg/otlp/metrics,go.uber.org/zap/internal/bufferpool,MIT,"Copyright (c) 2016-201
 pkg/otlp/metrics,go.uber.org/zap/internal/color,MIT,"Copyright (c) 2016-2017 Uber Technologies, Inc"
 pkg/otlp/metrics,go.uber.org/zap/internal/exit,MIT,"Copyright (c) 2016-2017 Uber Technologies, Inc"
 pkg/otlp/metrics,go.uber.org/zap/zapcore,MIT,"Copyright (c) 2016-2017 Uber Technologies, Inc"
+pkg/otlp/metrics,golang.org/x/exp/constraints,BSD-3-Clause,Copyright (c) 2009 The Go Authors. All rights reserved
+pkg/otlp/metrics,golang.org/x/exp/slices,BSD-3-Clause,Copyright (c) 2009 The Go Authors. All rights reserved
 pkg/otlp/metrics,golang.org/x/net/http/httpguts,BSD-3-Clause,Copyright (c) 2009 The Go Authors. All rights reserved
 pkg/otlp/metrics,golang.org/x/net/http2,BSD-3-Clause,Copyright (c) 2009 The Go Authors. All rights reserved
 pkg/otlp/metrics,golang.org/x/net/http2/hpack,BSD-3-Clause,Copyright (c) 2009 The Go Authors. All rights reserved

diff --git a/pkg/otlp/metrics/go.mod b/pkg/otlp/metrics/go.mod
@@ -14,6 +14,7 @@ require (
 	github.com/stretchr/testify v1.8.2
 	go.opentelemetry.io/collector/pdata v1.0.0-rc9
 	go.uber.org/zap v1.24.0
+	golang.org/x/exp v0.0.0-20230321023759-10a507213a29
 )
 
 require (

diff --git a/pkg/otlp/metrics/go.sum b/pkg/otlp/metrics/go.sum
@@ -66,6 +66,8 @@ go.uber.org/zap v1.24.0/go.mod h1:2kMP+WWQ8aoFoedH3T2sq6iJ2yDWpHbP0f6MQbS9Gkg=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
+golang.org/x/exp v0.0.0-20230321023759-10a507213a29 h1:ooxPy7fPvB4kwsA2h+iBNHkAbp/4JxTSwCmvdjEYmug=
+golang.org/x/exp v0.0.0-20230321023759-10a507213a29/go.mod h1:CxIveKay+FTh1D0yPZemJVgC/95VzuuOLq5Qi4xnoYc=
 golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=

diff --git a/pkg/otlp/metrics/metrics_translator.go b/pkg/otlp/metrics/metrics_translator.go
@@ -25,6 +25,7 @@ import (
 	"go.opentelemetry.io/collector/pdata/pcommon"
 	"go.opentelemetry.io/collector/pdata/pmetric"
 	"go.uber.org/zap"
+	"golang.org/x/exp/slices"
 
 	"github.com/DataDog/opentelemetry-mapping-go/pkg/otlp/attributes"
 	"github.com/DataDog/opentelemetry-mapping-go/pkg/otlp/attributes/source"
@@ -33,63 +34,6 @@ import (
 	"github.com/DataDog/opentelemetry-mapping-go/pkg/quantile"
 )
 
-// runtimeMetricMapping defines the fields needed to map OTel runtime metrics to their equivalent
-// Datadog runtime metrics
-type runtimeMetricMapping struct {
-	mappedName     string // the Datadog runtime metric name
-	attribute      string // the name of the attribute this metric originates from
-	attributeValue string // the value of the above attribute that corresponds with this metric
-}
-
-// runtimeMetricsMappings defines the mappings from OTel runtime metric names to their
-// equivalent Datadog runtime metric names
-var runtimeMetricsMappings = map[string][]runtimeMetricMapping{
-	"process.runtime.go.goroutines":                        {{mappedName: "runtime.go.num_goroutine"}},
-	"process.runtime.go.cgo.calls":                         {{mappedName: "runtime.go.num_cgo_call"}},
-	"process.runtime.go.lookups":                           {{mappedName: "runtime.go.mem_stats.lookups"}},
-	"process.runtime.go.mem.heap_alloc":                    {{mappedName: "runtime.go.mem_stats.heap_alloc"}},
-	"process.runtime.go.mem.heap_sys":                      {{mappedName: "runtime.go.mem_stats.heap_sys"}},
-	"process.runtime.go.mem.heap_idle":                     {{mappedName: "runtime.go.mem_stats.heap_idle"}},
-	"process.runtime.go.mem.heap_inuse":                    {{mappedName: "runtime.go.mem_stats.heap_inuse"}},
-	"process.runtime.go.mem.heap_released":                 {{mappedName: "runtime.go.mem_stats.heap_released"}},
-	"process.runtime.go.mem.heap_objects":                  {{mappedName: "runtime.go.mem_stats.heap_objects"}},
-	"process.runtime.go.gc.pause_total_ns":                 {{mappedName: "runtime.go.mem_stats.pause_total_ns"}},
-	"process.runtime.go.gc.count":                          {{mappedName: "runtime.go.mem_stats.num_gc"}},
-	"process.runtime.dotnet.thread_pool.threads.count":     {{mappedName: "runtime.dotnet.threads.count"}},
-	"process.runtime.dotnet.monitor.lock_contention.count": {{mappedName: "runtime.dotnet.threads.contention_count"}},
-	"process.runtime.dotnet.exceptions.count":              {{mappedName: "runtime.dotnet.exceptions.count"}},
-	"process.runtime.dotnet.gc.heap.size": {{
-		mappedName:     "runtime.dotnet.gc.size.gen0",
-		attribute:      "generation",
-		attributeValue: "gen0",
-	}, {
-		mappedName:     "runtime.dotnet.gc.size.gen1",
-		attribute:      "generation",
-		attributeValue: "gen1",
-	}, {
-		mappedName:     "runtime.dotnet.gc.size.gen2",
-		attribute:      "generation",
-		attributeValue: "gen2",
-	}, {
-		mappedName:     "runtime.dotnet.gc.size.loh",
-		attribute:      "generation",
-		attributeValue: "loh",
-	}},
-	"process.runtime.dotnet.gc.collections.count": {{
-		mappedName:     "runtime.dotnet.gc.count.gen0",
-		attribute:      "generation",
-		attributeValue: "gen0",
-	}, {
-		mappedName:     "runtime.dotnet.gc.count.gen1",
-		attribute:      "generation",
-		attributeValue: "gen1",
-	}, {
-		mappedName:     "runtime.dotnet.gc.count.gen2",
-		attribute:      "generation",
-		attributeValue: "gen2",
-	}},
-}
-
 const (
 	metricName             string = "metric name"
 	errNoBucketsNoSumCount string = "no buckets mode and no send count sum are incompatible"
@@ -540,28 +484,91 @@ func (t *Translator) source(m pcommon.Map) (source.Source, error) {
 
 // mapGaugeRuntimeMetricWithAttributes maps the specified runtime metric from metric attributes into a new Gauge metric
 func mapGaugeRuntimeMetricWithAttributes(md pmetric.Metric, metricsArray pmetric.MetricSlice, mp runtimeMetricMapping) {
-	cp := metricsArray.AppendEmpty()
-	cp.SetEmptyGauge()
 	for i := 0; i < md.Gauge().DataPoints().Len(); i++ {
-		attribute, res := md.Gauge().DataPoints().At(i).Attributes().Get(mp.attribute)
-		if res && attribute.AsString() == mp.attributeValue {
-			md.Gauge().DataPoints().At(i).CopyTo(cp.Gauge().DataPoints().AppendEmpty())
+		matchesAttributes := true
+		for _, attribute := range mp.attributes {
+			attributeValue, res := md.Gauge().DataPoints().At(i).Attributes().Get(attribute.key)
+			if !res || !slices.Contains(attribute.values, attributeValue.AsString()) {
+				matchesAttributes = false
+				break
+			}
+		}
+		if matchesAttributes {
+			cp := metricsArray.AppendEmpty()
+			cp.SetEmptyGauge()
+			dataPoint := cp.Gauge().DataPoints().AppendEmpty()
+			md.Gauge().DataPoints().At(i).CopyTo(dataPoint)
+			dataPoint.Attributes().RemoveIf(func(s string, value pcommon.Value) bool {
+				for _, attribute := range mp.attributes {
+					if s == attribute.key {
+						return true
+					}
+				}
+				return false
+			})
 			cp.SetName(mp.mappedName)
 		}
 	}
 }
 
 // mapSumRuntimeMetricWithAttributes maps the specified runtime metric from metric attributes into a new Sum metric
 func mapSumRuntimeMetricWithAttributes(md pmetric.Metric, metricsArray pmetric.MetricSlice, mp runtimeMetricMapping) {
-	cp := metricsArray.AppendEmpty()
-	cp.SetEmptySum()
-	cp.Sum().SetAggregationTemporality(md.Sum().AggregationTemporality())
-	cp.Sum().SetIsMonotonic(md.Sum().IsMonotonic())
 	for i := 0; i < md.Sum().DataPoints().Len(); i++ {
-		attribute, res := md.Sum().DataPoints().At(i).Attributes().Get(mp.attribute)
-		if res && attribute.AsString() == mp.attributeValue {
-			md.Sum().DataPoints().At(i).CopyTo(cp.Sum().DataPoints().AppendEmpty())
+		matchesAttributes := true
+		for _, attribute := range mp.attributes {
+			attributeValue, res := md.Sum().DataPoints().At(i).Attributes().Get(attribute.key)
+			if !res || !slices.Contains(attribute.values, attributeValue.AsString()) {
+				matchesAttributes = false
+				break
+			}
+		}
+		if matchesAttributes {
+			cp := metricsArray.AppendEmpty()
+			cp.SetEmptySum()
+			cp.Sum().SetAggregationTemporality(md.Sum().AggregationTemporality())
+			cp.Sum().SetIsMonotonic(md.Sum().IsMonotonic())
+			dataPoint := cp.Sum().DataPoints().AppendEmpty()
+			md.Sum().DataPoints().At(i).CopyTo(dataPoint)
+			dataPoint.Attributes().RemoveIf(func(s string, value pcommon.Value) bool {
+				for _, attribute := range mp.attributes {
+					if s == attribute.key {
+						return true
+					}
+				}
+				return false
+			})
+			cp.SetName(mp.mappedName)
+		}
+	}
+}
+
+// mapHistogramRuntimeMetricWithAttributes maps the specified runtime metric from metric attributes into a new Histogram metric
+func mapHistogramRuntimeMetricWithAttributes(md pmetric.Metric, metricsArray pmetric.MetricSlice, mp runtimeMetricMapping) {
+	for i := 0; i < md.Histogram().DataPoints().Len(); i++ {
+		matchesAttributes := true
+		for _, attribute := range mp.attributes {
+			attributeValue, res := md.Histogram().DataPoints().At(i).Attributes().Get(attribute.key)
+			if !res || !slices.Contains(attribute.values, attributeValue.AsString()) {
+				matchesAttributes = false
+				break
+			}
+		}
+		if matchesAttributes {
+			cp := metricsArray.AppendEmpty()
+			cp.SetEmptyHistogram()
+			cp.Histogram().SetAggregationTemporality(md.Histogram().AggregationTemporality())
+			dataPoint := cp.Histogram().DataPoints().AppendEmpty()
+			md.Histogram().DataPoints().At(i).CopyTo(dataPoint)
+			dataPoint.Attributes().RemoveIf(func(s string, value pcommon.Value) bool {
+				for _, attribute := range mp.attributes {
+					if s == attribute.key {
+						return true
+					}
+				}
+				return false
+			})
 			cp.SetName(mp.mappedName)
+			break
 		}
 	}
 }
@@ -617,7 +624,7 @@ func (t *Translator) MapMetrics(ctx context.Context, md pmetric.Metrics, consume
 				md := metricsArray.At(k)
 				if v, ok := runtimeMetricsMappings[md.Name()]; ok {
 					for _, mp := range v {
-						if mp.attribute == "" {
+						if mp.attributes == nil {
 							// duplicate runtime metrics as Datadog runtime metrics
 							cp := metricsArray.AppendEmpty()
 							md.CopyTo(cp)
@@ -628,6 +635,8 @@ func (t *Translator) MapMetrics(ctx context.Context, md pmetric.Metrics, consume
 							mapSumRuntimeMetricWithAttributes(md, metricsArray, mp)
 						} else if md.Type() == pmetric.MetricTypeGauge {
 							mapGaugeRuntimeMetricWithAttributes(md, metricsArray, mp)
+						} else if md.Type() == pmetric.MetricTypeHistogram {
+							mapHistogramRuntimeMetricWithAttributes(md, metricsArray, mp)
 						}
 					}
 				}