diff --git a/LICENSE-3rdparty.csv b/LICENSE-3rdparty.csv index 8e1f0689..7c4f123d 100644 --- a/LICENSE-3rdparty.csv +++ b/LICENSE-3rdparty.csv @@ -203,6 +203,8 @@ pkg/otlp/metrics,go.uber.org/zap/internal/bufferpool,MIT,"Copyright (c) 2016-201 pkg/otlp/metrics,go.uber.org/zap/internal/color,MIT,"Copyright (c) 2016-2017 Uber Technologies, Inc" pkg/otlp/metrics,go.uber.org/zap/internal/exit,MIT,"Copyright (c) 2016-2017 Uber Technologies, Inc" pkg/otlp/metrics,go.uber.org/zap/zapcore,MIT,"Copyright (c) 2016-2017 Uber Technologies, Inc" +pkg/otlp/metrics,golang.org/x/exp/constraints,BSD-3-Clause,Copyright (c) 2009 The Go Authors. All rights reserved +pkg/otlp/metrics,golang.org/x/exp/slices,BSD-3-Clause,Copyright (c) 2009 The Go Authors. All rights reserved pkg/otlp/metrics,golang.org/x/net/http/httpguts,BSD-3-Clause,Copyright (c) 2009 The Go Authors. All rights reserved pkg/otlp/metrics,golang.org/x/net/http2,BSD-3-Clause,Copyright (c) 2009 The Go Authors. All rights reserved pkg/otlp/metrics,golang.org/x/net/http2/hpack,BSD-3-Clause,Copyright (c) 2009 The Go Authors. All rights reserved diff --git a/pkg/otlp/metrics/go.mod b/pkg/otlp/metrics/go.mod index b2543b6a..9ea78df8 100644 --- a/pkg/otlp/metrics/go.mod +++ b/pkg/otlp/metrics/go.mod @@ -14,6 +14,7 @@ require ( github.com/stretchr/testify v1.8.2 go.opentelemetry.io/collector/pdata v1.0.0-rc9 go.uber.org/zap v1.24.0 + golang.org/x/exp v0.0.0-20230321023759-10a507213a29 ) require ( diff --git a/pkg/otlp/metrics/go.sum b/pkg/otlp/metrics/go.sum index fe4ab7c5..97178b41 100644 --- a/pkg/otlp/metrics/go.sum +++ b/pkg/otlp/metrics/go.sum @@ -66,6 +66,8 @@ go.uber.org/zap v1.24.0/go.mod h1:2kMP+WWQ8aoFoedH3T2sq6iJ2yDWpHbP0f6MQbS9Gkg= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/exp v0.0.0-20230321023759-10a507213a29 h1:ooxPy7fPvB4kwsA2h+iBNHkAbp/4JxTSwCmvdjEYmug= +golang.org/x/exp v0.0.0-20230321023759-10a507213a29/go.mod h1:CxIveKay+FTh1D0yPZemJVgC/95VzuuOLq5Qi4xnoYc= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= diff --git a/pkg/otlp/metrics/metrics_translator.go b/pkg/otlp/metrics/metrics_translator.go index 2e6d649d..13375cac 100644 --- a/pkg/otlp/metrics/metrics_translator.go +++ b/pkg/otlp/metrics/metrics_translator.go @@ -25,6 +25,7 @@ import ( "go.opentelemetry.io/collector/pdata/pcommon" "go.opentelemetry.io/collector/pdata/pmetric" "go.uber.org/zap" + "golang.org/x/exp/slices" "github.com/DataDog/opentelemetry-mapping-go/pkg/otlp/attributes" "github.com/DataDog/opentelemetry-mapping-go/pkg/otlp/attributes/source" @@ -33,63 +34,6 @@ import ( "github.com/DataDog/opentelemetry-mapping-go/pkg/quantile" ) -// runtimeMetricMapping defines the fields needed to map OTel runtime metrics to their equivalent -// Datadog runtime metrics -type runtimeMetricMapping struct { - mappedName string // the Datadog runtime metric name - attribute string // the name of the attribute this metric originates from - attributeValue string // the value of the above attribute that corresponds with this metric -} - -// runtimeMetricsMappings defines the mappings from OTel runtime metric names to their -// equivalent Datadog runtime metric names -var runtimeMetricsMappings = map[string][]runtimeMetricMapping{ - "process.runtime.go.goroutines": {{mappedName: "runtime.go.num_goroutine"}}, - "process.runtime.go.cgo.calls": {{mappedName: "runtime.go.num_cgo_call"}}, - "process.runtime.go.lookups": {{mappedName: "runtime.go.mem_stats.lookups"}}, - "process.runtime.go.mem.heap_alloc": {{mappedName: "runtime.go.mem_stats.heap_alloc"}}, - "process.runtime.go.mem.heap_sys": {{mappedName: "runtime.go.mem_stats.heap_sys"}}, - "process.runtime.go.mem.heap_idle": {{mappedName: "runtime.go.mem_stats.heap_idle"}}, - "process.runtime.go.mem.heap_inuse": {{mappedName: "runtime.go.mem_stats.heap_inuse"}}, - "process.runtime.go.mem.heap_released": {{mappedName: "runtime.go.mem_stats.heap_released"}}, - "process.runtime.go.mem.heap_objects": {{mappedName: "runtime.go.mem_stats.heap_objects"}}, - "process.runtime.go.gc.pause_total_ns": {{mappedName: "runtime.go.mem_stats.pause_total_ns"}}, - "process.runtime.go.gc.count": {{mappedName: "runtime.go.mem_stats.num_gc"}}, - "process.runtime.dotnet.thread_pool.threads.count": {{mappedName: "runtime.dotnet.threads.count"}}, - "process.runtime.dotnet.monitor.lock_contention.count": {{mappedName: "runtime.dotnet.threads.contention_count"}}, - "process.runtime.dotnet.exceptions.count": {{mappedName: "runtime.dotnet.exceptions.count"}}, - "process.runtime.dotnet.gc.heap.size": {{ - mappedName: "runtime.dotnet.gc.size.gen0", - attribute: "generation", - attributeValue: "gen0", - }, { - mappedName: "runtime.dotnet.gc.size.gen1", - attribute: "generation", - attributeValue: "gen1", - }, { - mappedName: "runtime.dotnet.gc.size.gen2", - attribute: "generation", - attributeValue: "gen2", - }, { - mappedName: "runtime.dotnet.gc.size.loh", - attribute: "generation", - attributeValue: "loh", - }}, - "process.runtime.dotnet.gc.collections.count": {{ - mappedName: "runtime.dotnet.gc.count.gen0", - attribute: "generation", - attributeValue: "gen0", - }, { - mappedName: "runtime.dotnet.gc.count.gen1", - attribute: "generation", - attributeValue: "gen1", - }, { - mappedName: "runtime.dotnet.gc.count.gen2", - attribute: "generation", - attributeValue: "gen2", - }}, -} - const ( metricName string = "metric name" errNoBucketsNoSumCount string = "no buckets mode and no send count sum are incompatible" @@ -540,12 +484,28 @@ func (t *Translator) source(m pcommon.Map) (source.Source, error) { // mapGaugeRuntimeMetricWithAttributes maps the specified runtime metric from metric attributes into a new Gauge metric func mapGaugeRuntimeMetricWithAttributes(md pmetric.Metric, metricsArray pmetric.MetricSlice, mp runtimeMetricMapping) { - cp := metricsArray.AppendEmpty() - cp.SetEmptyGauge() for i := 0; i < md.Gauge().DataPoints().Len(); i++ { - attribute, res := md.Gauge().DataPoints().At(i).Attributes().Get(mp.attribute) - if res && attribute.AsString() == mp.attributeValue { - md.Gauge().DataPoints().At(i).CopyTo(cp.Gauge().DataPoints().AppendEmpty()) + matchesAttributes := true + for _, attribute := range mp.attributes { + attributeValue, res := md.Gauge().DataPoints().At(i).Attributes().Get(attribute.key) + if !res || !slices.Contains(attribute.values, attributeValue.AsString()) { + matchesAttributes = false + break + } + } + if matchesAttributes { + cp := metricsArray.AppendEmpty() + cp.SetEmptyGauge() + dataPoint := cp.Gauge().DataPoints().AppendEmpty() + md.Gauge().DataPoints().At(i).CopyTo(dataPoint) + dataPoint.Attributes().RemoveIf(func(s string, value pcommon.Value) bool { + for _, attribute := range mp.attributes { + if s == attribute.key { + return true + } + } + return false + }) cp.SetName(mp.mappedName) } } @@ -553,15 +513,62 @@ func mapGaugeRuntimeMetricWithAttributes(md pmetric.Metric, metricsArray pmetric // mapSumRuntimeMetricWithAttributes maps the specified runtime metric from metric attributes into a new Sum metric func mapSumRuntimeMetricWithAttributes(md pmetric.Metric, metricsArray pmetric.MetricSlice, mp runtimeMetricMapping) { - cp := metricsArray.AppendEmpty() - cp.SetEmptySum() - cp.Sum().SetAggregationTemporality(md.Sum().AggregationTemporality()) - cp.Sum().SetIsMonotonic(md.Sum().IsMonotonic()) for i := 0; i < md.Sum().DataPoints().Len(); i++ { - attribute, res := md.Sum().DataPoints().At(i).Attributes().Get(mp.attribute) - if res && attribute.AsString() == mp.attributeValue { - md.Sum().DataPoints().At(i).CopyTo(cp.Sum().DataPoints().AppendEmpty()) + matchesAttributes := true + for _, attribute := range mp.attributes { + attributeValue, res := md.Sum().DataPoints().At(i).Attributes().Get(attribute.key) + if !res || !slices.Contains(attribute.values, attributeValue.AsString()) { + matchesAttributes = false + break + } + } + if matchesAttributes { + cp := metricsArray.AppendEmpty() + cp.SetEmptySum() + cp.Sum().SetAggregationTemporality(md.Sum().AggregationTemporality()) + cp.Sum().SetIsMonotonic(md.Sum().IsMonotonic()) + dataPoint := cp.Sum().DataPoints().AppendEmpty() + md.Sum().DataPoints().At(i).CopyTo(dataPoint) + dataPoint.Attributes().RemoveIf(func(s string, value pcommon.Value) bool { + for _, attribute := range mp.attributes { + if s == attribute.key { + return true + } + } + return false + }) + cp.SetName(mp.mappedName) + } + } +} + +// mapHistogramRuntimeMetricWithAttributes maps the specified runtime metric from metric attributes into a new Histogram metric +func mapHistogramRuntimeMetricWithAttributes(md pmetric.Metric, metricsArray pmetric.MetricSlice, mp runtimeMetricMapping) { + for i := 0; i < md.Histogram().DataPoints().Len(); i++ { + matchesAttributes := true + for _, attribute := range mp.attributes { + attributeValue, res := md.Histogram().DataPoints().At(i).Attributes().Get(attribute.key) + if !res || !slices.Contains(attribute.values, attributeValue.AsString()) { + matchesAttributes = false + break + } + } + if matchesAttributes { + cp := metricsArray.AppendEmpty() + cp.SetEmptyHistogram() + cp.Histogram().SetAggregationTemporality(md.Histogram().AggregationTemporality()) + dataPoint := cp.Histogram().DataPoints().AppendEmpty() + md.Histogram().DataPoints().At(i).CopyTo(dataPoint) + dataPoint.Attributes().RemoveIf(func(s string, value pcommon.Value) bool { + for _, attribute := range mp.attributes { + if s == attribute.key { + return true + } + } + return false + }) cp.SetName(mp.mappedName) + break } } } @@ -617,7 +624,7 @@ func (t *Translator) MapMetrics(ctx context.Context, md pmetric.Metrics, consume md := metricsArray.At(k) if v, ok := runtimeMetricsMappings[md.Name()]; ok { for _, mp := range v { - if mp.attribute == "" { + if mp.attributes == nil { // duplicate runtime metrics as Datadog runtime metrics cp := metricsArray.AppendEmpty() md.CopyTo(cp) @@ -628,6 +635,8 @@ func (t *Translator) MapMetrics(ctx context.Context, md pmetric.Metrics, consume mapSumRuntimeMetricWithAttributes(md, metricsArray, mp) } else if md.Type() == pmetric.MetricTypeGauge { mapGaugeRuntimeMetricWithAttributes(md, metricsArray, mp) + } else if md.Type() == pmetric.MetricTypeHistogram { + mapHistogramRuntimeMetricWithAttributes(md, metricsArray, mp) } } } diff --git a/pkg/otlp/metrics/metrics_translator_test.go b/pkg/otlp/metrics/metrics_translator_test.go index d8387ea8..83829950 100644 --- a/pkg/otlp/metrics/metrics_translator_test.go +++ b/pkg/otlp/metrics/metrics_translator_test.go @@ -103,6 +103,7 @@ func newTranslator(t *testing.T, logger *zap.Logger) *Translator { WithFallbackSourceProvider(testProvider(fallbackHostname)), WithHistogramMode(HistogramModeDistributions), WithNumberMode(NumberModeCumulativeToDelta), + WithHistogramAggregations(), } tr, err := NewTranslator( @@ -428,7 +429,11 @@ func TestMapSumRuntimeMetricWithAttributesHasMapping(t *testing.T) { ctx := context.Background() tr := newTranslator(t, zap.NewNop()) consumer := &mockFullConsumer{} - if err := tr.MapMetrics(ctx, createTestMetricWithAttributes(false, "process.runtime.dotnet.gc.collections.count", pmetric.MetricTypeSum, "generation", "gen"), consumer); err != nil { + attributes := []runtimeMetricAttribute{{ + key: "generation", + values: []string{"gen0"}, + }} + if err := tr.MapMetrics(ctx, createTestMetricWithAttributes("process.runtime.dotnet.gc.collections.count", pmetric.MetricTypeSum, attributes, 1), consumer); err != nil { t.Fatal(err) } startTs := int(getProcessStartTime()) + 1 @@ -436,9 +441,7 @@ func TestMapSumRuntimeMetricWithAttributesHasMapping(t *testing.T) { consumer.metrics, []metric{ newCountWithHost(newDims("process.runtime.dotnet.gc.collections.count").AddTags("generation:gen0"), uint64(seconds(startTs+1)), 10, fallbackHostname), - newCountWithHost(newDims("runtime.dotnet.gc.count.gen0").AddTags("generation:gen0"), uint64(seconds(startTs+1)), 10, fallbackHostname), - newCountWithHost(newDims("runtime.dotnet.gc.count.gen1").AddTags("generation:gen1"), uint64(seconds(startTs+2)), 15, fallbackHostname), - newCountWithHost(newDims("runtime.dotnet.gc.count.gen2").AddTags("generation:gen2"), uint64(seconds(startTs+3)), 20, fallbackHostname), + newCountWithHost(newDims("runtime.dotnet.gc.count.gen0"), uint64(seconds(startTs+1)), 10, fallbackHostname), }, ) } @@ -447,19 +450,147 @@ func TestMapGaugeRuntimeMetricWithAttributesHasMapping(t *testing.T) { ctx := context.Background() tr := newTranslator(t, zap.NewNop()) consumer := &mockFullConsumer{} - if err := tr.MapMetrics(ctx, createTestMetricWithAttributes(false, "process.runtime.dotnet.gc.heap.size", pmetric.MetricTypeGauge, "generation", "gen"), consumer); err != nil { + attributes := []runtimeMetricAttribute{{ + key: "generation", + values: []string{"gen1"}, + }} + if err := tr.MapMetrics(ctx, createTestMetricWithAttributes("process.runtime.dotnet.gc.heap.size", pmetric.MetricTypeGauge, attributes, 1), consumer); err != nil { t.Fatal(err) } startTs := int(getProcessStartTime()) + 1 assert.ElementsMatch(t, consumer.metrics, []metric{ - newGaugeWithHost(newDims("process.runtime.dotnet.gc.heap.size").AddTags("generation:gen0"), uint64(seconds(startTs+1)), 10, fallbackHostname), - newGaugeWithHost(newDims("process.runtime.dotnet.gc.heap.size").AddTags("generation:gen1"), uint64(seconds(startTs+2)), 15, fallbackHostname), - newGaugeWithHost(newDims("process.runtime.dotnet.gc.heap.size").AddTags("generation:gen2"), uint64(seconds(startTs+3)), 20, fallbackHostname), - newGaugeWithHost(newDims("runtime.dotnet.gc.size.gen0").AddTags("generation:gen0"), uint64(seconds(startTs+1)), 10, fallbackHostname), - newGaugeWithHost(newDims("runtime.dotnet.gc.size.gen1").AddTags("generation:gen1"), uint64(seconds(startTs+2)), 15, fallbackHostname), - newGaugeWithHost(newDims("runtime.dotnet.gc.size.gen2").AddTags("generation:gen2"), uint64(seconds(startTs+3)), 20, fallbackHostname), + newGaugeWithHost(newDims("process.runtime.dotnet.gc.heap.size").AddTags("generation:gen1"), uint64(seconds(startTs+1)), 10, fallbackHostname), + newGaugeWithHost(newDims("runtime.dotnet.gc.size.gen1"), uint64(seconds(startTs+1)), 10, fallbackHostname), + }, + ) +} + +func TestMapHistogramRuntimeMetricHasMapping(t *testing.T) { + ctx := context.Background() + tr := newTranslator(t, zap.NewNop()) + consumer := &mockFullConsumer{} + fmt.Println(consumer.metrics) + + if err := tr.MapMetrics(ctx, createTestHistogramMetric("process.runtime.jvm.gc.duration"), consumer); err != nil { + t.Fatal(err) + } + startTs := int(getProcessStartTime()) + 1 + assert.ElementsMatch(t, + consumer.metrics, + []metric{ + newCountWithHost(newDims("process.runtime.jvm.gc.duration.count"), uint64(seconds(startTs+1)), 100, fallbackHostname), + newCountWithHost(newDims("process.runtime.jvm.gc.duration.sum"), uint64(seconds(startTs+1)), 0, fallbackHostname), + newGaugeWithHost(newDims("process.runtime.jvm.gc.duration.min"), uint64(seconds(startTs+1)), -100, fallbackHostname), + newGaugeWithHost(newDims("process.runtime.jvm.gc.duration.max"), uint64(seconds(startTs+1)), 100, fallbackHostname), + newCountWithHost(newDims("jvm.gc.parnew.time.count"), uint64(seconds(startTs+1)), 100, fallbackHostname), + newCountWithHost(newDims("jvm.gc.parnew.time.sum"), uint64(seconds(startTs+1)), 0, fallbackHostname), + newGaugeWithHost(newDims("jvm.gc.parnew.time.min"), uint64(seconds(startTs+1)), -100, fallbackHostname), + newGaugeWithHost(newDims("jvm.gc.parnew.time.max"), uint64(seconds(startTs+1)), 100, fallbackHostname), + }, + ) +} + +func TestMapHistogramRuntimeMetricWithAttributesHasMapping(t *testing.T) { + ctx := context.Background() + tr := newTranslator(t, zap.NewNop()) + consumer := &mockFullConsumer{} + attributes := []runtimeMetricAttribute{{ + key: "generation", + values: []string{"gen1"}, + }} + if err := tr.MapMetrics(ctx, createTestMetricWithAttributes("process.runtime.dotnet.gc.heap.size", pmetric.MetricTypeHistogram, attributes, 1), consumer); err != nil { + t.Fatal(err) + } + startTs := int(getProcessStartTime()) + 1 + assert.ElementsMatch(t, + consumer.metrics, + []metric{ + newCountWithHost(newDims("process.runtime.dotnet.gc.heap.size.count").AddTags("generation:gen1"), uint64(seconds(startTs+1)), 100, fallbackHostname), + newCountWithHost(newDims("process.runtime.dotnet.gc.heap.size.sum").AddTags("generation:gen1"), uint64(seconds(startTs+1)), 0, fallbackHostname), + newGaugeWithHost(newDims("process.runtime.dotnet.gc.heap.size.min").AddTags("generation:gen1"), uint64(seconds(startTs+1)), -100, fallbackHostname), + newGaugeWithHost(newDims("process.runtime.dotnet.gc.heap.size.max").AddTags("generation:gen1"), uint64(seconds(startTs+1)), 100, fallbackHostname), + newCountWithHost(newDims("runtime.dotnet.gc.size.gen1.count"), uint64(seconds(startTs+1)), 100, fallbackHostname), + newCountWithHost(newDims("runtime.dotnet.gc.size.gen1.sum"), uint64(seconds(startTs+1)), 0, fallbackHostname), + newGaugeWithHost(newDims("runtime.dotnet.gc.size.gen1.min"), uint64(seconds(startTs+1)), -100, fallbackHostname), + newGaugeWithHost(newDims("runtime.dotnet.gc.size.gen1.max"), uint64(seconds(startTs+1)), 100, fallbackHostname), + }, + ) +} + +func TestMapRuntimeMetricWithTwoAttributesHasMapping(t *testing.T) { + ctx := context.Background() + tr := newTranslator(t, zap.NewNop()) + consumer := &mockFullConsumer{} + attributes := []runtimeMetricAttribute{{ + key: "pool", + values: []string{"G1 Old Gen"}, + }, { + key: "type", + values: []string{"heap"}, + }} + if err := tr.MapMetrics(ctx, createTestMetricWithAttributes("process.runtime.jvm.memory.usage", pmetric.MetricTypeGauge, attributes, 1), consumer); err != nil { + t.Fatal(err) + } + startTs := int(getProcessStartTime()) + 1 + assert.ElementsMatch(t, + consumer.metrics, + []metric{ + newGaugeWithHost(newDims("process.runtime.jvm.memory.usage").AddTags("pool:G1 Old Gen", "type:heap"), uint64(seconds(startTs+1)), 10, fallbackHostname), + newGaugeWithHost(newDims("jvm.heap_memory").AddTags("pool:G1 Old Gen"), uint64(seconds(startTs+1)), 10, fallbackHostname), + newGaugeWithHost(newDims("jvm.gc.old_gen_size"), uint64(seconds(startTs+1)), 10, fallbackHostname), + }, + ) +} + +func TestMapRuntimeMetricWithTwoAttributesMultipleDataPointsHasMapping(t *testing.T) { + ctx := context.Background() + tr := newTranslator(t, zap.NewNop()) + consumer := &mockFullConsumer{} + attributes := []runtimeMetricAttribute{{ + key: "pool", + values: []string{"G1 Old Gen", "G1 Survivor Space", "G1 Eden Space"}, + }, { + key: "type", + values: []string{"heap", "heap", "heap"}, + }} + if err := tr.MapMetrics(ctx, createTestMetricWithAttributes("process.runtime.jvm.memory.usage", pmetric.MetricTypeGauge, attributes, 3), consumer); err != nil { + t.Fatal(err) + } + startTs := int(getProcessStartTime()) + 1 + assert.ElementsMatch(t, + consumer.metrics, + []metric{ + newGaugeWithHost(newDims("process.runtime.jvm.memory.usage").AddTags("pool:G1 Old Gen", "type:heap"), uint64(seconds(startTs+1)), 10, fallbackHostname), + newGaugeWithHost(newDims("process.runtime.jvm.memory.usage").AddTags("pool:G1 Survivor Space", "type:heap"), uint64(seconds(startTs+2)), 20, fallbackHostname), + newGaugeWithHost(newDims("process.runtime.jvm.memory.usage").AddTags("pool:G1 Eden Space", "type:heap"), uint64(seconds(startTs+3)), 30, fallbackHostname), + newGaugeWithHost(newDims("jvm.heap_memory").AddTags("pool:G1 Old Gen"), uint64(seconds(startTs+1)), 10, fallbackHostname), + newGaugeWithHost(newDims("jvm.heap_memory").AddTags("pool:G1 Survivor Space"), uint64(seconds(startTs+2)), 20, fallbackHostname), + newGaugeWithHost(newDims("jvm.heap_memory").AddTags("pool:G1 Eden Space"), uint64(seconds(startTs+3)), 30, fallbackHostname), + newGaugeWithHost(newDims("jvm.gc.old_gen_size"), uint64(seconds(startTs+1)), 10, fallbackHostname), + newGaugeWithHost(newDims("jvm.gc.survivor_size"), uint64(seconds(startTs+2)), 20, fallbackHostname), + newGaugeWithHost(newDims("jvm.gc.eden_size"), uint64(seconds(startTs+3)), 30, fallbackHostname), + }, + ) +} + +func TestMapGaugeRuntimeMetricWithInvalidAttributes(t *testing.T) { + ctx := context.Background() + tr := newTranslator(t, zap.NewNop()) + consumer := &mockFullConsumer{} + attributes := []runtimeMetricAttribute{{ + key: "type", + values: []string{"heap2"}, + }} + if err := tr.MapMetrics(ctx, createTestMetricWithAttributes("process.runtime.jvm.memory.usage", pmetric.MetricTypeGauge, attributes, 1), consumer); err != nil { + t.Fatal(err) + } + startTs := int(getProcessStartTime()) + 1 + assert.ElementsMatch(t, + consumer.metrics, + []metric{ + newGaugeWithHost(newDims("process.runtime.jvm.memory.usage").AddTags("type:heap2"), uint64(seconds(startTs+1)), 10, fallbackHostname), }, ) } @@ -827,11 +958,34 @@ func createTestDoubleCumulativeMonotonicMetrics(tsmatch bool) pmetric.Metrics { return md } -func createTestMetricWithAttributes(tsmatch bool, metricName string, metricType pmetric.MetricType, attribute string, attributeValue string) pmetric.Metrics { +func createTestHistogramMetric(metricName string) pmetric.Metrics { + md := pmetric.NewMetrics() + met := md.ResourceMetrics().AppendEmpty().ScopeMetrics().AppendEmpty().Metrics().AppendEmpty() + met.SetName(metricName) + var hpsCount pmetric.HistogramDataPointSlice + met.SetEmptyHistogram() + met.Histogram().SetAggregationTemporality(pmetric.AggregationTemporalityDelta) + hpsCount = met.Histogram().DataPoints() + hpsCount.EnsureCapacity(1) + startTs := int(getProcessStartTime()) + 1 + hpCount := hpsCount.AppendEmpty() + hpCount.SetStartTimestamp(seconds(startTs)) + hpCount.SetTimestamp(seconds(startTs + 1)) + hpCount.ExplicitBounds().FromRaw([]float64{}) + hpCount.BucketCounts().FromRaw([]uint64{100}) + hpCount.SetCount(100) + hpCount.SetSum(0) + hpCount.SetMin(-100) + hpCount.SetMax(100) + return md +} + +func createTestMetricWithAttributes(metricName string, metricType pmetric.MetricType, attributes []runtimeMetricAttribute, dataPoints int) pmetric.Metrics { md := pmetric.NewMetrics() met := md.ResourceMetrics().AppendEmpty().ScopeMetrics().AppendEmpty().Metrics().AppendEmpty() met.SetName(metricName) var dpsInt pmetric.NumberDataPointSlice + var hpsCount pmetric.HistogramDataPointSlice if metricType == pmetric.MetricTypeSum { met.SetEmptySum() met.Sum().SetAggregationTemporality(pmetric.AggregationTemporalityCumulative) @@ -840,20 +994,42 @@ func createTestMetricWithAttributes(tsmatch bool, metricName string, metricType } else if metricType == pmetric.MetricTypeGauge { met.SetEmptyGauge() dpsInt = met.Gauge().DataPoints() + } else if metricType == pmetric.MetricTypeHistogram { + met.SetEmptyHistogram() + met.Histogram().SetAggregationTemporality(pmetric.AggregationTemporalityDelta) + hpsCount = met.Histogram().DataPoints() } - values := []int64{10, 15, 20} - dpsInt.EnsureCapacity(len(values)) - startTs := int(getProcessStartTime()) + 1 - for i, val := range values { - dpInt := dpsInt.AppendEmpty() - dpInt.Attributes().PutStr(attribute, fmt.Sprintf("%v%v", attributeValue, i)) - dpInt.SetStartTimestamp(seconds(startTs)) - if tsmatch { - dpInt.SetTimestamp(seconds(startTs)) - } else { - dpInt.SetTimestamp(seconds(startTs + i + 1)) + + if metricType != pmetric.MetricTypeHistogram { + dpsInt.EnsureCapacity(dataPoints) + for i := 0; i < dataPoints; i++ { + startTs := int(getProcessStartTime()) + 1 + dpInt := dpsInt.AppendEmpty() + for _, attr := range attributes { + dpInt.Attributes().PutStr(attr.key, attr.values[i]) + } + dpInt.SetStartTimestamp(seconds(startTs)) + dpInt.SetTimestamp(seconds(startTs + 1 + i)) + dpInt.SetIntValue(int64(10 * (1 + i))) } - dpInt.SetIntValue(val) + return md + } + + hpsCount.EnsureCapacity(dataPoints) + for i := 0; i < dataPoints; i++ { + startTs := int(getProcessStartTime()) + 1 + hpCount := hpsCount.AppendEmpty() + for _, attr := range attributes { + hpCount.Attributes().PutStr(attr.key, attr.values[i]) + } + hpCount.SetStartTimestamp(seconds(startTs)) + hpCount.SetTimestamp(seconds(startTs + 1 + i)) + hpCount.ExplicitBounds().FromRaw([]float64{}) + hpCount.BucketCounts().FromRaw([]uint64{100}) + hpCount.SetCount(100) + hpCount.SetSum(0) + hpCount.SetMin(-100) + hpCount.SetMax(100) } return md } diff --git a/pkg/otlp/metrics/runtime_metric_mappings.go b/pkg/otlp/metrics/runtime_metric_mappings.go new file mode 100644 index 00000000..b751a391 --- /dev/null +++ b/pkg/otlp/metrics/runtime_metric_mappings.go @@ -0,0 +1,237 @@ +package metrics + +// runtimeMetricMapping defines the fields needed to map OTel runtime metrics to their equivalent +// Datadog runtime metrics +type runtimeMetricMapping struct { + mappedName string // the Datadog runtime metric name + attributes []runtimeMetricAttribute // the attribute(s) this metric originates from +} + +// runtimeMetricAttribute defines the structure for an attribute in regard to mapping runtime metrics. +// The presence of a runtimeMetricAttribute means that a metric must be mapped from a data point +// with the given attribute(s). +type runtimeMetricAttribute struct { + key string // the attribute name + values []string // the attribute value, or multiple values if there is more than one value for the same mapping +} + +// runtimeMetricMappingList defines the structure for a list of runtime metric mappings where the key +// represents the OTel metric name and the runtimeMetricMapping contains the Datadog metric name +type runtimeMetricMappingList map[string][]runtimeMetricMapping + +var goRuntimeMetricsMappings = runtimeMetricMappingList{ + "process.runtime.go.goroutines": {{mappedName: "runtime.go.num_goroutine"}}, + "process.runtime.go.cgo.calls": {{mappedName: "runtime.go.num_cgo_call"}}, + "process.runtime.go.lookups": {{mappedName: "runtime.go.mem_stats.lookups"}}, + "process.runtime.go.mem.heap_alloc": {{mappedName: "runtime.go.mem_stats.heap_alloc"}}, + "process.runtime.go.mem.heap_sys": {{mappedName: "runtime.go.mem_stats.heap_sys"}}, + "process.runtime.go.mem.heap_idle": {{mappedName: "runtime.go.mem_stats.heap_idle"}}, + "process.runtime.go.mem.heap_inuse": {{mappedName: "runtime.go.mem_stats.heap_inuse"}}, + "process.runtime.go.mem.heap_released": {{mappedName: "runtime.go.mem_stats.heap_released"}}, + "process.runtime.go.mem.heap_objects": {{mappedName: "runtime.go.mem_stats.heap_objects"}}, + "process.runtime.go.gc.pause_total_ns": {{mappedName: "runtime.go.mem_stats.pause_total_ns"}}, + "process.runtime.go.gc.count": {{mappedName: "runtime.go.mem_stats.num_gc"}}, +} + +var dotnetRuntimeMetricsMappings = runtimeMetricMappingList{ + "process.runtime.dotnet.thread_pool.threads.count": {{mappedName: "runtime.dotnet.threads.count"}}, + "process.runtime.dotnet.monitor.lock_contention.count": {{mappedName: "runtime.dotnet.threads.contention_count"}}, + "process.runtime.dotnet.exceptions.count": {{mappedName: "runtime.dotnet.exceptions.count"}}, + "process.runtime.dotnet.gc.heap.size": {{ + mappedName: "runtime.dotnet.gc.size.gen0", + attributes: []runtimeMetricAttribute{{ + key: "generation", + values: []string{"gen0"}, + }}, + }, { + mappedName: "runtime.dotnet.gc.size.gen1", + attributes: []runtimeMetricAttribute{{ + key: "generation", + values: []string{"gen1"}, + }}, + }, { + mappedName: "runtime.dotnet.gc.size.gen2", + attributes: []runtimeMetricAttribute{{ + key: "generation", + values: []string{"gen2"}, + }}, + }, { + mappedName: "runtime.dotnet.gc.size.loh", + attributes: []runtimeMetricAttribute{{ + key: "generation", + values: []string{"loh"}, + }}, + }}, + "process.runtime.dotnet.gc.collections.count": {{ + mappedName: "runtime.dotnet.gc.count.gen0", + attributes: []runtimeMetricAttribute{{ + key: "generation", + values: []string{"gen0"}, + }}, + }, { + mappedName: "runtime.dotnet.gc.count.gen1", + attributes: []runtimeMetricAttribute{{ + key: "generation", + values: []string{"gen1"}, + }}, + }, { + mappedName: "runtime.dotnet.gc.count.gen2", + attributes: []runtimeMetricAttribute{{ + key: "generation", + values: []string{"gen2"}, + }}, + }}, +} + +var javaRuntimeMetricsMappings = runtimeMetricMappingList{ + "process.runtime.jvm.threads.count": {{mappedName: "jvm.thread_count"}}, + "process.runtime.jvm.classes.loaded": {{mappedName: "jvm.loaded_classes"}}, + "process.runtime.jvm.system.cpu.utilization": {{mappedName: "jvm.cpu_load.system"}}, + "process.runtime.jvm.cpu.utilization": {{mappedName: "jvm.cpu_load.process"}}, + "process.runtime.jvm.gc.duration": {{mappedName: "jvm.gc.parnew.time"}}, + "process.runtime.jvm.memory.usage": {{ + mappedName: "jvm.heap_memory", + attributes: []runtimeMetricAttribute{{ + key: "type", + values: []string{"heap"}, + }}, + }, { + mappedName: "jvm.non_heap_memory", + attributes: []runtimeMetricAttribute{{ + key: "type", + values: []string{"non_heap"}, + }}, + }, { + mappedName: "jvm.gc.old_gen_size", + attributes: []runtimeMetricAttribute{{ + key: "pool", + values: []string{"G1 Old Gen", "Tenured Gen", "PS Old Gen"}, + }, { + key: "type", + values: []string{"heap"}, + }}, + }, { + mappedName: "jvm.gc.eden_size", + attributes: []runtimeMetricAttribute{{ + key: "pool", + values: []string{"G1 Eden Space", "Eden Space", "Par Eden Space", "PS Eden Space"}, + }, { + key: "type", + values: []string{"heap"}, + }}, + }, { + mappedName: "jvm.gc.survivor_size", + attributes: []runtimeMetricAttribute{{ + key: "pool", + values: []string{"G1 Survivor Space", "Survivor Space", "Par Survivor Space", "PS Survivor Space"}, + }, { + key: "type", + values: []string{"heap"}, + }}, + }, { + mappedName: "jvm.gc.metaspace_size", + attributes: []runtimeMetricAttribute{{ + key: "pool", + values: []string{"Metaspace"}, + }, { + key: "type", + values: []string{"non_heap"}, + }}, + }}, + "process.runtime.jvm.memory.committed": {{ + mappedName: "jvm.heap_memory_committed", + attributes: []runtimeMetricAttribute{{ + key: "type", + values: []string{"heap"}, + }}, + }, { + mappedName: "jvm.non_heap_memory_committed", + attributes: []runtimeMetricAttribute{{ + key: "type", + values: []string{"non_heap"}, + }}, + }}, + "process.runtime.jvm.memory.init": {{ + mappedName: "jvm.heap_memory_init", + attributes: []runtimeMetricAttribute{{ + key: "type", + values: []string{"heap"}, + }}, + }, { + mappedName: "jvm.non_heap_memory_init", + attributes: []runtimeMetricAttribute{{ + key: "type", + values: []string{"non_heap"}, + }}, + }}, + "process.runtime.jvm.memory.limit": {{ + mappedName: "jvm.heap_memory_max", + attributes: []runtimeMetricAttribute{{ + key: "type", + values: []string{"heap"}, + }}, + }, { + mappedName: "jvm.non_heap_memory_max", + attributes: []runtimeMetricAttribute{{ + key: "type", + values: []string{"non_heap"}, + }}, + }}, + "process.runtime.jvm.buffer.usage": {{ + mappedName: "jvm.buffer_pool.direct.used", + attributes: []runtimeMetricAttribute{{ + key: "pool", + values: []string{"direct"}, + }}, + }, { + mappedName: "jvm.buffer_pool.mapped.used", + attributes: []runtimeMetricAttribute{{ + key: "pool", + values: []string{"mapped"}, + }}, + }}, + "process.runtime.jvm.buffer.count": {{ + mappedName: "jvm.buffer_pool.direct.count", + attributes: []runtimeMetricAttribute{{ + key: "pool", + values: []string{"direct"}, + }}, + }, { + mappedName: "jvm.buffer_pool.mapped.count", + attributes: []runtimeMetricAttribute{{ + key: "pool", + values: []string{"mapped"}, + }}, + }}, + "process.runtime.jvm.buffer.limit": {{ + mappedName: "jvm.buffer_pool.direct.limit", + attributes: []runtimeMetricAttribute{{ + key: "pool", + values: []string{"direct"}, + }}, + }, { + mappedName: "jvm.buffer_pool.mapped.limit", + attributes: []runtimeMetricAttribute{{ + key: "pool", + values: []string{"mapped"}, + }}, + }}, +} + +func getRuntimeMetricsMappings() runtimeMetricMappingList { + res := runtimeMetricMappingList{} + for k, v := range goRuntimeMetricsMappings { + res[k] = v + } + for k, v := range dotnetRuntimeMetricsMappings { + res[k] = v + } + for k, v := range javaRuntimeMetricsMappings { + res[k] = v + } + return res +} + +// runtimeMetricsMappings defines the mappings from OTel runtime metric names to their +// equivalent Datadog runtime metric names +var runtimeMetricsMappings = getRuntimeMetricsMappings()