From 1311115be4662e31a409e6101b8a6a8dbd83b773 Mon Sep 17 00:00:00 2001 From: Stanley Liu Date: Wed, 1 Mar 2023 14:49:39 -0500 Subject: [PATCH 01/22] Add java mappings --- pkg/otlp/metrics/metrics_translator.go | 46 ++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) diff --git a/pkg/otlp/metrics/metrics_translator.go b/pkg/otlp/metrics/metrics_translator.go index b6156483..b1e801a4 100644 --- a/pkg/otlp/metrics/metrics_translator.go +++ b/pkg/otlp/metrics/metrics_translator.go @@ -96,6 +96,52 @@ var runtimeMetricsMappings = map[string][]runtimeMetricMapping{ attributeValue: "gen2", metricType: pmetric.MetricTypeSum, }}, + "process.runtime.jvm.threads.count": {{mappedName: "jvm.thread_count"}}, + "process.runtime.jvm.gc.duration": {{mappedName: "jvm.gc.parnew.time"}}, + "process.runtime.jvm.memory.usage": {{ + mappedName: "jvm.heap_memory", + attribute: "type", + attributeValue: "heap", + metricType: pmetric.MetricTypeGauge, + }, { + mappedName: "jvm.non_heap_memory", + attribute: "type", + attributeValue: "nonheap", + metricType: pmetric.MetricTypeGauge, + }}, + "process.runtime.jvm.memory.committed": {{ + mappedName: "jvm.heap_memory_committed", + attribute: "type", + attributeValue: "heap", + metricType: pmetric.MetricTypeGauge, + }, { + mappedName: "jvm.non_heap_memory_committed", + attribute: "type", + attributeValue: "nonheap", + metricType: pmetric.MetricTypeGauge, + }}, + "process.runtime.jvm.memory.init": {{ + mappedName: "jvm.heap_memory_init", + attribute: "type", + attributeValue: "heap", + metricType: pmetric.MetricTypeGauge, + }, { + mappedName: "jvm.non_heap_memory_init", + attribute: "type", + attributeValue: "nonheap", + metricType: pmetric.MetricTypeGauge, + }}, + "process.runtime.jvm.memory.limit": {{ + mappedName: "jvm.heap_memory_max", + attribute: "type", + attributeValue: "heap", + metricType: pmetric.MetricTypeGauge, + }, { + mappedName: "jvm.non_heap_memory_max", + attribute: "type", + attributeValue: "nonheap", + metricType: pmetric.MetricTypeGauge, + }}, } const metricName string = "metric name" From ac8084eab7d98487d3d9c34b640b47b49c477fc8 Mon Sep 17 00:00:00 2001 From: Stanley Liu Date: Wed, 1 Mar 2023 14:57:59 -0500 Subject: [PATCH 02/22] Debug logs --- pkg/otlp/metrics/metrics_translator.go | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/pkg/otlp/metrics/metrics_translator.go b/pkg/otlp/metrics/metrics_translator.go index b1e801a4..2131933c 100644 --- a/pkg/otlp/metrics/metrics_translator.go +++ b/pkg/otlp/metrics/metrics_translator.go @@ -548,6 +548,9 @@ func (t *Translator) source(m pcommon.Map) (source.Source, error) { // mapGaugeRuntimeMetricWithAttributes maps the specified runtime metric from metric attributes into a new Gauge metric func mapGaugeRuntimeMetricWithAttributes(md pmetric.Metric, metricsArray pmetric.MetricSlice, mp runtimeMetricMapping) { + fmt.Println("IN MAP GAUGE") + fmt.Printf("--------- md.name: %v --------\n", md.Name()) + fmt.Printf("--------- mp: %+v --------\n", mp) cp := metricsArray.AppendEmpty() cp.SetEmptyGauge() for i := 0; i < md.Gauge().DataPoints().Len(); i++ { @@ -561,6 +564,9 @@ func mapGaugeRuntimeMetricWithAttributes(md pmetric.Metric, metricsArray pmetric // mapSumRuntimeMetricWithAttributes maps the specified runtime metric from metric attributes into a new Sum metric func mapSumRuntimeMetricWithAttributes(md pmetric.Metric, metricsArray pmetric.MetricSlice, mp runtimeMetricMapping) { + fmt.Println("IN MAP SUM") + fmt.Printf("--------- md.name: %v --------\n", md.Name()) + fmt.Printf("--------- mp: %+v --------\n", mp) cp := metricsArray.AppendEmpty() cp.SetEmptySum() cp.Sum().SetAggregationTemporality(md.Sum().AggregationTemporality()) @@ -624,6 +630,9 @@ func (t *Translator) MapMetrics(ctx context.Context, md pmetric.Metrics, consume for k := 0; k < metricsArray.Len(); k++ { md := metricsArray.At(k) if v, ok := runtimeMetricsMappings[md.Name()]; ok { + fmt.Println("IN MAP METRICS") + fmt.Printf("--------- md.name: %v --------\n", md.Name()) + for _, mp := range v { if mp.attribute == "" { // duplicate runtime metrics as Datadog runtime metrics From 30a7441e983a929c185d6958bc98ce3b3b66df76 Mon Sep 17 00:00:00 2001 From: Stanley Liu Date: Wed, 1 Mar 2023 15:05:09 -0500 Subject: [PATCH 03/22] More Debug logs --- pkg/otlp/metrics/metrics_translator.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/pkg/otlp/metrics/metrics_translator.go b/pkg/otlp/metrics/metrics_translator.go index 2131933c..2ea30ae4 100644 --- a/pkg/otlp/metrics/metrics_translator.go +++ b/pkg/otlp/metrics/metrics_translator.go @@ -550,7 +550,12 @@ func (t *Translator) source(m pcommon.Map) (source.Source, error) { func mapGaugeRuntimeMetricWithAttributes(md pmetric.Metric, metricsArray pmetric.MetricSlice, mp runtimeMetricMapping) { fmt.Println("IN MAP GAUGE") fmt.Printf("--------- md.name: %v --------\n", md.Name()) + fmt.Printf("--------- md.Type(): %v --------\n", md.Type()) fmt.Printf("--------- mp: %+v --------\n", mp) + fmt.Printf("--------- md.Gauge(): %v --------\n", md.Gauge()) + fmt.Printf("--------- md.Gauge().DataPoints(): %v --------\n", md.Gauge().DataPoints()) + fmt.Printf("--------- md.Gauge().DataPoints().At(0): %v --------\n", md.Gauge().DataPoints().At(0)) + fmt.Printf("--------- md.Gauge().DataPoints().Len(): %v --------\n", md.Gauge().DataPoints().Len()) cp := metricsArray.AppendEmpty() cp.SetEmptyGauge() for i := 0; i < md.Gauge().DataPoints().Len(); i++ { From fbc42c38cebf703c712a7ce4957b4ab0e5a7915f Mon Sep 17 00:00:00 2001 From: Stanley Liu Date: Wed, 1 Mar 2023 15:08:15 -0500 Subject: [PATCH 04/22] Change to use md type --- pkg/otlp/metrics/metrics_translator.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/otlp/metrics/metrics_translator.go b/pkg/otlp/metrics/metrics_translator.go index 2ea30ae4..4b506800 100644 --- a/pkg/otlp/metrics/metrics_translator.go +++ b/pkg/otlp/metrics/metrics_translator.go @@ -646,9 +646,9 @@ func (t *Translator) MapMetrics(ctx context.Context, md pmetric.Metrics, consume cp.SetName(mp.mappedName) break } - if mp.metricType == pmetric.MetricTypeSum { + if md.Type() == pmetric.MetricTypeSum { mapSumRuntimeMetricWithAttributes(md, metricsArray, mp) - } else if mp.metricType == pmetric.MetricTypeGauge { + } else if md.Type() == pmetric.MetricTypeGauge { mapGaugeRuntimeMetricWithAttributes(md, metricsArray, mp) } } From 714ac65151acc4e2ad6b023efbead4adf2d7d39a Mon Sep 17 00:00:00 2001 From: Stanley Liu Date: Wed, 1 Mar 2023 15:13:25 -0500 Subject: [PATCH 05/22] Change to non_heap and remove metricType --- pkg/otlp/metrics/metrics_translator.go | 38 +++----------------------- 1 file changed, 4 insertions(+), 34 deletions(-) diff --git a/pkg/otlp/metrics/metrics_translator.go b/pkg/otlp/metrics/metrics_translator.go index 4b506800..0f1c6d77 100644 --- a/pkg/otlp/metrics/metrics_translator.go +++ b/pkg/otlp/metrics/metrics_translator.go @@ -39,7 +39,6 @@ type runtimeMetricMapping struct { mappedName string // the Datadog runtime metric name attribute string // the name of the attribute this metric originates from attributeValue string // the value of the above attribute that corresponds with this metric - metricType pmetric.MetricType } // runtimeMetricsMappings defines the mappings from OTel runtime metric names to their @@ -63,38 +62,31 @@ var runtimeMetricsMappings = map[string][]runtimeMetricMapping{ mappedName: "runtime.dotnet.gc.size.gen0", attribute: "generation", attributeValue: "gen0", - metricType: pmetric.MetricTypeGauge, }, { mappedName: "runtime.dotnet.gc.size.gen1", attribute: "generation", attributeValue: "gen1", - metricType: pmetric.MetricTypeGauge, }, { mappedName: "runtime.dotnet.gc.size.gen2", attribute: "generation", attributeValue: "gen2", - metricType: pmetric.MetricTypeGauge, }, { mappedName: "runtime.dotnet.gc.size.loh", attribute: "generation", attributeValue: "loh", - metricType: pmetric.MetricTypeGauge, }}, "process.runtime.dotnet.gc.collections.count": {{ mappedName: "runtime.dotnet.gc.count.gen0", attribute: "generation", attributeValue: "gen0", - metricType: pmetric.MetricTypeSum, }, { mappedName: "runtime.dotnet.gc.count.gen1", attribute: "generation", attributeValue: "gen1", - metricType: pmetric.MetricTypeSum, }, { mappedName: "runtime.dotnet.gc.count.gen2", attribute: "generation", attributeValue: "gen2", - metricType: pmetric.MetricTypeSum, }}, "process.runtime.jvm.threads.count": {{mappedName: "jvm.thread_count"}}, "process.runtime.jvm.gc.duration": {{mappedName: "jvm.gc.parnew.time"}}, @@ -102,45 +94,37 @@ var runtimeMetricsMappings = map[string][]runtimeMetricMapping{ mappedName: "jvm.heap_memory", attribute: "type", attributeValue: "heap", - metricType: pmetric.MetricTypeGauge, }, { mappedName: "jvm.non_heap_memory", attribute: "type", - attributeValue: "nonheap", - metricType: pmetric.MetricTypeGauge, + attributeValue: "non_heap", }}, "process.runtime.jvm.memory.committed": {{ mappedName: "jvm.heap_memory_committed", attribute: "type", attributeValue: "heap", - metricType: pmetric.MetricTypeGauge, }, { mappedName: "jvm.non_heap_memory_committed", attribute: "type", - attributeValue: "nonheap", - metricType: pmetric.MetricTypeGauge, + attributeValue: "non_heap", }}, "process.runtime.jvm.memory.init": {{ mappedName: "jvm.heap_memory_init", attribute: "type", attributeValue: "heap", - metricType: pmetric.MetricTypeGauge, }, { mappedName: "jvm.non_heap_memory_init", attribute: "type", - attributeValue: "nonheap", - metricType: pmetric.MetricTypeGauge, + attributeValue: "non_heap", }}, "process.runtime.jvm.memory.limit": {{ mappedName: "jvm.heap_memory_max", attribute: "type", attributeValue: "heap", - metricType: pmetric.MetricTypeGauge, }, { mappedName: "jvm.non_heap_memory_max", attribute: "type", - attributeValue: "nonheap", - metricType: pmetric.MetricTypeGauge, + attributeValue: "non_heap", }}, } @@ -548,14 +532,6 @@ func (t *Translator) source(m pcommon.Map) (source.Source, error) { // mapGaugeRuntimeMetricWithAttributes maps the specified runtime metric from metric attributes into a new Gauge metric func mapGaugeRuntimeMetricWithAttributes(md pmetric.Metric, metricsArray pmetric.MetricSlice, mp runtimeMetricMapping) { - fmt.Println("IN MAP GAUGE") - fmt.Printf("--------- md.name: %v --------\n", md.Name()) - fmt.Printf("--------- md.Type(): %v --------\n", md.Type()) - fmt.Printf("--------- mp: %+v --------\n", mp) - fmt.Printf("--------- md.Gauge(): %v --------\n", md.Gauge()) - fmt.Printf("--------- md.Gauge().DataPoints(): %v --------\n", md.Gauge().DataPoints()) - fmt.Printf("--------- md.Gauge().DataPoints().At(0): %v --------\n", md.Gauge().DataPoints().At(0)) - fmt.Printf("--------- md.Gauge().DataPoints().Len(): %v --------\n", md.Gauge().DataPoints().Len()) cp := metricsArray.AppendEmpty() cp.SetEmptyGauge() for i := 0; i < md.Gauge().DataPoints().Len(); i++ { @@ -569,9 +545,6 @@ func mapGaugeRuntimeMetricWithAttributes(md pmetric.Metric, metricsArray pmetric // mapSumRuntimeMetricWithAttributes maps the specified runtime metric from metric attributes into a new Sum metric func mapSumRuntimeMetricWithAttributes(md pmetric.Metric, metricsArray pmetric.MetricSlice, mp runtimeMetricMapping) { - fmt.Println("IN MAP SUM") - fmt.Printf("--------- md.name: %v --------\n", md.Name()) - fmt.Printf("--------- mp: %+v --------\n", mp) cp := metricsArray.AppendEmpty() cp.SetEmptySum() cp.Sum().SetAggregationTemporality(md.Sum().AggregationTemporality()) @@ -635,9 +608,6 @@ func (t *Translator) MapMetrics(ctx context.Context, md pmetric.Metrics, consume for k := 0; k < metricsArray.Len(); k++ { md := metricsArray.At(k) if v, ok := runtimeMetricsMappings[md.Name()]; ok { - fmt.Println("IN MAP METRICS") - fmt.Printf("--------- md.name: %v --------\n", md.Name()) - for _, mp := range v { if mp.attribute == "" { // duplicate runtime metrics as Datadog runtime metrics From 367294d4461f019f77e493cee6a3b20b32249dc1 Mon Sep 17 00:00:00 2001 From: Stanley Liu Date: Fri, 3 Mar 2023 14:24:10 -0500 Subject: [PATCH 06/22] Add unit test --- pkg/otlp/metrics/metrics_translator_test.go | 36 +++++++++++++++++---- 1 file changed, 30 insertions(+), 6 deletions(-) diff --git a/pkg/otlp/metrics/metrics_translator_test.go b/pkg/otlp/metrics/metrics_translator_test.go index d8387ea8..8e03a0ce 100644 --- a/pkg/otlp/metrics/metrics_translator_test.go +++ b/pkg/otlp/metrics/metrics_translator_test.go @@ -16,7 +16,6 @@ package metrics import ( "context" - "fmt" "math" "testing" "time" @@ -428,7 +427,7 @@ func TestMapSumRuntimeMetricWithAttributesHasMapping(t *testing.T) { ctx := context.Background() tr := newTranslator(t, zap.NewNop()) consumer := &mockFullConsumer{} - if err := tr.MapMetrics(ctx, createTestMetricWithAttributes(false, "process.runtime.dotnet.gc.collections.count", pmetric.MetricTypeSum, "generation", "gen"), consumer); err != nil { + if err := tr.MapMetrics(ctx, createTestMetricWithAttributes(false, "process.runtime.dotnet.gc.collections.count", pmetric.MetricTypeSum, "generation", []string{"gen0", "gen1", "gen2"}), consumer); err != nil { t.Fatal(err) } startTs := int(getProcessStartTime()) + 1 @@ -447,7 +446,7 @@ func TestMapGaugeRuntimeMetricWithAttributesHasMapping(t *testing.T) { ctx := context.Background() tr := newTranslator(t, zap.NewNop()) consumer := &mockFullConsumer{} - if err := tr.MapMetrics(ctx, createTestMetricWithAttributes(false, "process.runtime.dotnet.gc.heap.size", pmetric.MetricTypeGauge, "generation", "gen"), consumer); err != nil { + if err := tr.MapMetrics(ctx, createTestMetricWithAttributes(false, "process.runtime.dotnet.gc.heap.size", pmetric.MetricTypeGauge, "generation", []string{"gen0", "gen1", "gen2"}), consumer); err != nil { t.Fatal(err) } startTs := int(getProcessStartTime()) + 1 @@ -464,6 +463,27 @@ func TestMapGaugeRuntimeMetricWithAttributesHasMapping(t *testing.T) { ) } +func TestMapGaugeRuntimeMetricWithInvalidAttributes(t *testing.T) { + ctx := context.Background() + tr := newTranslator(t, zap.NewNop()) + consumer := &mockFullConsumer{} + if err := tr.MapMetrics(ctx, createTestMetricWithAttributes(false, "process.runtime.jvm.memory.usage", pmetric.MetricTypeGauge, "type", []string{"heap", "heap1", "heap2", "non_heap"}), consumer); err != nil { + t.Fatal(err) + } + startTs := int(getProcessStartTime()) + 1 + assert.ElementsMatch(t, + consumer.metrics, + []metric{ + newGaugeWithHost(newDims("process.runtime.jvm.memory.usage").AddTags("type:heap"), uint64(seconds(startTs+1)), 10, fallbackHostname), + newGaugeWithHost(newDims("process.runtime.jvm.memory.usage").AddTags("type:heap1"), uint64(seconds(startTs+2)), 15, fallbackHostname), + newGaugeWithHost(newDims("process.runtime.jvm.memory.usage").AddTags("type:heap2"), uint64(seconds(startTs+3)), 20, fallbackHostname), + newGaugeWithHost(newDims("process.runtime.jvm.memory.usage").AddTags("type:non_heap"), uint64(seconds(startTs+4)), 25, fallbackHostname), + newGaugeWithHost(newDims("jvm.heap_memory").AddTags("type:heap"), uint64(seconds(startTs+1)), 10, fallbackHostname), + newGaugeWithHost(newDims("jvm.non_heap_memory").AddTags("type:non_heap"), uint64(seconds(startTs+4)), 25, fallbackHostname), + }, + ) +} + func TestMapRuntimeMetricsNoMapping(t *testing.T) { ctx := context.Background() tr := newTranslator(t, zap.NewNop()) @@ -827,7 +847,7 @@ func createTestDoubleCumulativeMonotonicMetrics(tsmatch bool) pmetric.Metrics { return md } -func createTestMetricWithAttributes(tsmatch bool, metricName string, metricType pmetric.MetricType, attribute string, attributeValue string) pmetric.Metrics { +func createTestMetricWithAttributes(tsmatch bool, metricName string, metricType pmetric.MetricType, attribute string, attributeValues []string) pmetric.Metrics { md := pmetric.NewMetrics() met := md.ResourceMetrics().AppendEmpty().ScopeMetrics().AppendEmpty().Metrics().AppendEmpty() met.SetName(metricName) @@ -841,12 +861,16 @@ func createTestMetricWithAttributes(tsmatch bool, metricName string, metricType met.SetEmptyGauge() dpsInt = met.Gauge().DataPoints() } - values := []int64{10, 15, 20} + + var values []int64 + for i := range attributeValues { + values = append(values, int64(10+(i*5))) + } dpsInt.EnsureCapacity(len(values)) startTs := int(getProcessStartTime()) + 1 for i, val := range values { dpInt := dpsInt.AppendEmpty() - dpInt.Attributes().PutStr(attribute, fmt.Sprintf("%v%v", attributeValue, i)) + dpInt.Attributes().PutStr(attribute, attributeValues[i]) dpInt.SetStartTimestamp(seconds(startTs)) if tsmatch { dpInt.SetTimestamp(seconds(startTs)) From d87dbf82cc1c6c1b3f4742f9158817e1d873c80f Mon Sep 17 00:00:00 2001 From: Stanley Liu Date: Mon, 6 Mar 2023 11:48:37 -0500 Subject: [PATCH 07/22] Refactor runtime metric mappings --- pkg/otlp/metrics/metrics_translator.go | 50 ++++++++++++++++++-------- 1 file changed, 36 insertions(+), 14 deletions(-) diff --git a/pkg/otlp/metrics/metrics_translator.go b/pkg/otlp/metrics/metrics_translator.go index 0f1c6d77..9911c198 100644 --- a/pkg/otlp/metrics/metrics_translator.go +++ b/pkg/otlp/metrics/metrics_translator.go @@ -41,20 +41,21 @@ type runtimeMetricMapping struct { attributeValue string // the value of the above attribute that corresponds with this metric } -// runtimeMetricsMappings defines the mappings from OTel runtime metric names to their -// equivalent Datadog runtime metric names -var runtimeMetricsMappings = map[string][]runtimeMetricMapping{ - "process.runtime.go.goroutines": {{mappedName: "runtime.go.num_goroutine"}}, - "process.runtime.go.cgo.calls": {{mappedName: "runtime.go.num_cgo_call"}}, - "process.runtime.go.lookups": {{mappedName: "runtime.go.mem_stats.lookups"}}, - "process.runtime.go.mem.heap_alloc": {{mappedName: "runtime.go.mem_stats.heap_alloc"}}, - "process.runtime.go.mem.heap_sys": {{mappedName: "runtime.go.mem_stats.heap_sys"}}, - "process.runtime.go.mem.heap_idle": {{mappedName: "runtime.go.mem_stats.heap_idle"}}, - "process.runtime.go.mem.heap_inuse": {{mappedName: "runtime.go.mem_stats.heap_inuse"}}, - "process.runtime.go.mem.heap_released": {{mappedName: "runtime.go.mem_stats.heap_released"}}, - "process.runtime.go.mem.heap_objects": {{mappedName: "runtime.go.mem_stats.heap_objects"}}, - "process.runtime.go.gc.pause_total_ns": {{mappedName: "runtime.go.mem_stats.pause_total_ns"}}, - "process.runtime.go.gc.count": {{mappedName: "runtime.go.mem_stats.num_gc"}}, +var goRuntimeMetricsMappings = map[string][]runtimeMetricMapping{ + "process.runtime.go.goroutines": {{mappedName: "runtime.go.num_goroutine"}}, + "process.runtime.go.cgo.calls": {{mappedName: "runtime.go.num_cgo_call"}}, + "process.runtime.go.lookups": {{mappedName: "runtime.go.mem_stats.lookups"}}, + "process.runtime.go.mem.heap_alloc": {{mappedName: "runtime.go.mem_stats.heap_alloc"}}, + "process.runtime.go.mem.heap_sys": {{mappedName: "runtime.go.mem_stats.heap_sys"}}, + "process.runtime.go.mem.heap_idle": {{mappedName: "runtime.go.mem_stats.heap_idle"}}, + "process.runtime.go.mem.heap_inuse": {{mappedName: "runtime.go.mem_stats.heap_inuse"}}, + "process.runtime.go.mem.heap_released": {{mappedName: "runtime.go.mem_stats.heap_released"}}, + "process.runtime.go.mem.heap_objects": {{mappedName: "runtime.go.mem_stats.heap_objects"}}, + "process.runtime.go.gc.pause_total_ns": {{mappedName: "runtime.go.mem_stats.pause_total_ns"}}, + "process.runtime.go.gc.count": {{mappedName: "runtime.go.mem_stats.num_gc"}}, +} + +var dotnetRuntimeMetricsMappings = map[string][]runtimeMetricMapping{ "process.runtime.dotnet.thread_pool.threads.count": {{mappedName: "runtime.dotnet.threads.count"}}, "process.runtime.dotnet.monitor.lock_contention.count": {{mappedName: "runtime.dotnet.threads.contention_count"}}, "process.runtime.dotnet.exceptions.count": {{mappedName: "runtime.dotnet.exceptions.count"}}, @@ -88,6 +89,9 @@ var runtimeMetricsMappings = map[string][]runtimeMetricMapping{ attribute: "generation", attributeValue: "gen2", }}, +} + +var javaRuntimeMetricsMappings = map[string][]runtimeMetricMapping{ "process.runtime.jvm.threads.count": {{mappedName: "jvm.thread_count"}}, "process.runtime.jvm.gc.duration": {{mappedName: "jvm.gc.parnew.time"}}, "process.runtime.jvm.memory.usage": {{ @@ -128,6 +132,24 @@ var runtimeMetricsMappings = map[string][]runtimeMetricMapping{ }}, } +func getRuntimeMetricsMappings() map[string][]runtimeMetricMapping { + res := map[string][]runtimeMetricMapping{} + for k, v := range goRuntimeMetricsMappings { + res[k] = v + } + for k, v := range dotnetRuntimeMetricsMappings { + res[k] = v + } + for k, v := range javaRuntimeMetricsMappings { + res[k] = v + } + return res +} + +// runtimeMetricsMappings defines the mappings from OTel runtime metric names to their +// equivalent Datadog runtime metric names +var runtimeMetricsMappings = getRuntimeMetricsMappings() + const metricName string = "metric name" var _ source.Provider = (*noSourceProvider)(nil) From 9e66fa7065edc0bb5b1f54ed01334203977a05fb Mon Sep 17 00:00:00 2001 From: Stanley Liu Date: Tue, 4 Apr 2023 13:59:36 -0400 Subject: [PATCH 08/22] Change runtime metric attribute definition --- pkg/otlp/metrics/metrics_translator.go | 142 ++++++++++++++++--------- 1 file changed, 94 insertions(+), 48 deletions(-) diff --git a/pkg/otlp/metrics/metrics_translator.go b/pkg/otlp/metrics/metrics_translator.go index bb058f03..6ee51705 100644 --- a/pkg/otlp/metrics/metrics_translator.go +++ b/pkg/otlp/metrics/metrics_translator.go @@ -36,9 +36,16 @@ import ( // runtimeMetricMapping defines the fields needed to map OTel runtime metrics to their equivalent // Datadog runtime metrics type runtimeMetricMapping struct { - mappedName string // the Datadog runtime metric name - attribute string // the name of the attribute this metric originates from - attributeValue string // the value of the above attribute that corresponds with this metric + mappedName string // the Datadog runtime metric name + attributes []runtimeMetricAttribute // the attribute(s) this metric originates from +} + +// runtimeMetricAttribute defines the structure for an attribute in regard to mapping runtime metrics. +// The presence of a runtimeMetricAttribute means that a metric must be mapped from a data point +// with the given attribute(s). +type runtimeMetricAttribute struct { + key string // the attribute name + values []string // the attribute value, or multiple values if there is more than one value for the same mapping } var goRuntimeMetricsMappings = map[string][]runtimeMetricMapping{ @@ -60,34 +67,48 @@ var dotnetRuntimeMetricsMappings = map[string][]runtimeMetricMapping{ "process.runtime.dotnet.monitor.lock_contention.count": {{mappedName: "runtime.dotnet.threads.contention_count"}}, "process.runtime.dotnet.exceptions.count": {{mappedName: "runtime.dotnet.exceptions.count"}}, "process.runtime.dotnet.gc.heap.size": {{ - mappedName: "runtime.dotnet.gc.size.gen0", - attribute: "generation", - attributeValue: "gen0", + mappedName: "runtime.dotnet.gc.size.gen0", + attributes: []runtimeMetricAttribute{{ + key: "generation", + values: []string{"gen0"}, + }}, }, { - mappedName: "runtime.dotnet.gc.size.gen1", - attribute: "generation", - attributeValue: "gen1", + mappedName: "runtime.dotnet.gc.size.gen1", + attributes: []runtimeMetricAttribute{{ + key: "generation", + values: []string{"gen1"}, + }}, }, { - mappedName: "runtime.dotnet.gc.size.gen2", - attribute: "generation", - attributeValue: "gen2", + mappedName: "runtime.dotnet.gc.size.gen2", + attributes: []runtimeMetricAttribute{{ + key: "generation", + values: []string{"gen2"}, + }}, }, { - mappedName: "runtime.dotnet.gc.size.loh", - attribute: "generation", - attributeValue: "loh", + mappedName: "runtime.dotnet.gc.size.loh", + attributes: []runtimeMetricAttribute{{ + key: "generation", + values: []string{"loh"}, + }}, }}, "process.runtime.dotnet.gc.collections.count": {{ - mappedName: "runtime.dotnet.gc.count.gen0", - attribute: "generation", - attributeValue: "gen0", + mappedName: "runtime.dotnet.gc.count.gen0", + attributes: []runtimeMetricAttribute{{ + key: "generation", + values: []string{"gen0"}, + }}, }, { - mappedName: "runtime.dotnet.gc.count.gen1", - attribute: "generation", - attributeValue: "gen1", + mappedName: "runtime.dotnet.gc.count.gen1", + attributes: []runtimeMetricAttribute{{ + key: "generation", + values: []string{"gen1"}, + }}, }, { - mappedName: "runtime.dotnet.gc.count.gen2", - attribute: "generation", - attributeValue: "gen2", + mappedName: "runtime.dotnet.gc.count.gen2", + attributes: []runtimeMetricAttribute{{ + key: "generation", + values: []string{"gen2"}, + }}, }}, } @@ -95,40 +116,65 @@ var javaRuntimeMetricsMappings = map[string][]runtimeMetricMapping{ "process.runtime.jvm.threads.count": {{mappedName: "jvm.thread_count"}}, "process.runtime.jvm.gc.duration": {{mappedName: "jvm.gc.parnew.time"}}, "process.runtime.jvm.memory.usage": {{ - mappedName: "jvm.heap_memory", - attribute: "type", - attributeValue: "heap", + mappedName: "jvm.heap_memory", + attributes: []runtimeMetricAttribute{{ + key: "type", + values: []string{"heap"}, + }}, + }, { + mappedName: "jvm.non_heap_memory", + attributes: []runtimeMetricAttribute{{ + key: "type", + values: []string{"non_heap"}, + }}, }, { - mappedName: "jvm.non_heap_memory", - attribute: "type", - attributeValue: "non_heap", + mappedName: "jvm.gc.old_gen_size", + attributes: []runtimeMetricAttribute{{ + key: "pool", + values: []string{"g1_old_gen", "ps_old_gen", "tenured_gen"}, + }, { + key: "type", + values: []string{"heap"}, + }}, }}, "process.runtime.jvm.memory.committed": {{ - mappedName: "jvm.heap_memory_committed", - attribute: "type", - attributeValue: "heap", + mappedName: "jvm.heap_memory_committed", + attributes: []runtimeMetricAttribute{{ + key: "type", + values: []string{"heap"}, + }}, }, { - mappedName: "jvm.non_heap_memory_committed", - attribute: "type", - attributeValue: "non_heap", + mappedName: "jvm.non_heap_memory_committed", + attributes: []runtimeMetricAttribute{{ + key: "type", + values: []string{"non_heap"}, + }}, }}, "process.runtime.jvm.memory.init": {{ - mappedName: "jvm.heap_memory_init", - attribute: "type", - attributeValue: "heap", + mappedName: "jvm.heap_memory_init", + attributes: []runtimeMetricAttribute{{ + key: "type", + values: []string{"heap"}, + }}, }, { - mappedName: "jvm.non_heap_memory_init", - attribute: "type", - attributeValue: "non_heap", + mappedName: "jvm.non_heap_memory_init", + attributes: []runtimeMetricAttribute{{ + key: "type", + values: []string{"non_heap"}, + }}, }}, "process.runtime.jvm.memory.limit": {{ - mappedName: "jvm.heap_memory_max", - attribute: "type", - attributeValue: "heap", + mappedName: "jvm.heap_memory_max", + attributes: []runtimeMetricAttribute{{ + key: "type", + values: []string{"heap"}, + }}, }, { - mappedName: "jvm.non_heap_memory_max", - attribute: "type", - attributeValue: "non_heap", + mappedName: "jvm.non_heap_memory_max", + attributes: []runtimeMetricAttribute{{ + key: "type", + values: []string{"non_heap"}, + }}, }}, } From 9ab5c8142360b2dc1fe4f9ed1f327ba87d61c75d Mon Sep 17 00:00:00 2001 From: Stanley Liu Date: Tue, 4 Apr 2023 14:01:02 -0400 Subject: [PATCH 09/22] Handle metrics with multiple attributes, add code to remove mapping attributes --- pkg/otlp/metrics/go.mod | 1 + pkg/otlp/metrics/go.sum | 2 + pkg/otlp/metrics/metrics_translator.go | 61 +++++++++++--- pkg/otlp/metrics/metrics_translator_test.go | 93 +++++++++++++-------- 4 files changed, 109 insertions(+), 48 deletions(-) diff --git a/pkg/otlp/metrics/go.mod b/pkg/otlp/metrics/go.mod index 556c7949..22a5d4a3 100644 --- a/pkg/otlp/metrics/go.mod +++ b/pkg/otlp/metrics/go.mod @@ -14,6 +14,7 @@ require ( github.com/stretchr/testify v1.8.2 go.opentelemetry.io/collector/pdata v1.0.0-rc5 go.uber.org/zap v1.24.0 + golang.org/x/exp v0.0.0-20230321023759-10a507213a29 ) require ( diff --git a/pkg/otlp/metrics/go.sum b/pkg/otlp/metrics/go.sum index 9f37c99f..86b7d269 100644 --- a/pkg/otlp/metrics/go.sum +++ b/pkg/otlp/metrics/go.sum @@ -65,6 +65,8 @@ go.uber.org/zap v1.24.0/go.mod h1:2kMP+WWQ8aoFoedH3T2sq6iJ2yDWpHbP0f6MQbS9Gkg= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/exp v0.0.0-20230321023759-10a507213a29 h1:ooxPy7fPvB4kwsA2h+iBNHkAbp/4JxTSwCmvdjEYmug= +golang.org/x/exp v0.0.0-20230321023759-10a507213a29/go.mod h1:CxIveKay+FTh1D0yPZemJVgC/95VzuuOLq5Qi4xnoYc= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= diff --git a/pkg/otlp/metrics/metrics_translator.go b/pkg/otlp/metrics/metrics_translator.go index 6ee51705..46220cd5 100644 --- a/pkg/otlp/metrics/metrics_translator.go +++ b/pkg/otlp/metrics/metrics_translator.go @@ -25,6 +25,7 @@ import ( "go.opentelemetry.io/collector/pdata/pcommon" "go.opentelemetry.io/collector/pdata/pmetric" "go.uber.org/zap" + "golang.org/x/exp/slices" "github.com/DataDog/opentelemetry-mapping-go/pkg/otlp/attributes" "github.com/DataDog/opentelemetry-mapping-go/pkg/otlp/attributes/source" @@ -614,28 +615,62 @@ func (t *Translator) source(m pcommon.Map) (source.Source, error) { // mapGaugeRuntimeMetricWithAttributes maps the specified runtime metric from metric attributes into a new Gauge metric func mapGaugeRuntimeMetricWithAttributes(md pmetric.Metric, metricsArray pmetric.MetricSlice, mp runtimeMetricMapping) { - cp := metricsArray.AppendEmpty() - cp.SetEmptyGauge() for i := 0; i < md.Gauge().DataPoints().Len(); i++ { - attribute, res := md.Gauge().DataPoints().At(i).Attributes().Get(mp.attribute) - if res && attribute.AsString() == mp.attributeValue { - md.Gauge().DataPoints().At(i).CopyTo(cp.Gauge().DataPoints().AppendEmpty()) + matchesAttributes := true + for _, attribute := range mp.attributes { + attributeValue, res := md.Gauge().DataPoints().At(i).Attributes().Get(attribute.key) + if !res || !slices.Contains(attribute.values, attributeValue.AsString()) { + matchesAttributes = false + break + } + } + if matchesAttributes { + cp := metricsArray.AppendEmpty() + cp.SetEmptyGauge() + dataPoint := cp.Gauge().DataPoints().AppendEmpty() + md.Gauge().DataPoints().At(i).CopyTo(dataPoint) + dataPoint.Attributes().RemoveIf(func(s string, value pcommon.Value) bool { + for _, attribute := range mp.attributes { + if s == attribute.key { + return true + } + } + return false + }) cp.SetName(mp.mappedName) + break } } } // mapSumRuntimeMetricWithAttributes maps the specified runtime metric from metric attributes into a new Sum metric func mapSumRuntimeMetricWithAttributes(md pmetric.Metric, metricsArray pmetric.MetricSlice, mp runtimeMetricMapping) { - cp := metricsArray.AppendEmpty() - cp.SetEmptySum() - cp.Sum().SetAggregationTemporality(md.Sum().AggregationTemporality()) - cp.Sum().SetIsMonotonic(md.Sum().IsMonotonic()) for i := 0; i < md.Sum().DataPoints().Len(); i++ { - attribute, res := md.Sum().DataPoints().At(i).Attributes().Get(mp.attribute) - if res && attribute.AsString() == mp.attributeValue { - md.Sum().DataPoints().At(i).CopyTo(cp.Sum().DataPoints().AppendEmpty()) + matchesAttributes := true + for _, attribute := range mp.attributes { + attributeValue, res := md.Sum().DataPoints().At(i).Attributes().Get(attribute.key) + if !res || !slices.Contains(attribute.values, attributeValue.AsString()) { + matchesAttributes = false + break + } + } + if matchesAttributes { + cp := metricsArray.AppendEmpty() + cp.SetEmptySum() + cp.Sum().SetAggregationTemporality(md.Sum().AggregationTemporality()) + cp.Sum().SetIsMonotonic(md.Sum().IsMonotonic()) + dataPoint := cp.Sum().DataPoints().AppendEmpty() + md.Sum().DataPoints().At(i).CopyTo(dataPoint) + dataPoint.Attributes().RemoveIf(func(s string, value pcommon.Value) bool { + for _, attribute := range mp.attributes { + if s == attribute.key { + return true + } + } + return false + }) cp.SetName(mp.mappedName) + break } } } @@ -691,7 +726,7 @@ func (t *Translator) MapMetrics(ctx context.Context, md pmetric.Metrics, consume md := metricsArray.At(k) if v, ok := runtimeMetricsMappings[md.Name()]; ok { for _, mp := range v { - if mp.attribute == "" { + if mp.attributes == nil { // duplicate runtime metrics as Datadog runtime metrics cp := metricsArray.AppendEmpty() md.CopyTo(cp) diff --git a/pkg/otlp/metrics/metrics_translator_test.go b/pkg/otlp/metrics/metrics_translator_test.go index 8e03a0ce..d0d47b10 100644 --- a/pkg/otlp/metrics/metrics_translator_test.go +++ b/pkg/otlp/metrics/metrics_translator_test.go @@ -427,7 +427,11 @@ func TestMapSumRuntimeMetricWithAttributesHasMapping(t *testing.T) { ctx := context.Background() tr := newTranslator(t, zap.NewNop()) consumer := &mockFullConsumer{} - if err := tr.MapMetrics(ctx, createTestMetricWithAttributes(false, "process.runtime.dotnet.gc.collections.count", pmetric.MetricTypeSum, "generation", []string{"gen0", "gen1", "gen2"}), consumer); err != nil { + attributes := []runtimeMetricAttribute{{ + key: "generation", + values: []string{"gen0"}, + }} + if err := tr.MapMetrics(ctx, createTestMetricWithAttributes(false, "process.runtime.dotnet.gc.collections.count", pmetric.MetricTypeSum, attributes), consumer); err != nil { t.Fatal(err) } startTs := int(getProcessStartTime()) + 1 @@ -435,9 +439,7 @@ func TestMapSumRuntimeMetricWithAttributesHasMapping(t *testing.T) { consumer.metrics, []metric{ newCountWithHost(newDims("process.runtime.dotnet.gc.collections.count").AddTags("generation:gen0"), uint64(seconds(startTs+1)), 10, fallbackHostname), - newCountWithHost(newDims("runtime.dotnet.gc.count.gen0").AddTags("generation:gen0"), uint64(seconds(startTs+1)), 10, fallbackHostname), - newCountWithHost(newDims("runtime.dotnet.gc.count.gen1").AddTags("generation:gen1"), uint64(seconds(startTs+2)), 15, fallbackHostname), - newCountWithHost(newDims("runtime.dotnet.gc.count.gen2").AddTags("generation:gen2"), uint64(seconds(startTs+3)), 20, fallbackHostname), + newCountWithHost(newDims("runtime.dotnet.gc.count.gen0"), uint64(seconds(startTs+1)), 10, fallbackHostname), }, ) } @@ -446,44 +448,69 @@ func TestMapGaugeRuntimeMetricWithAttributesHasMapping(t *testing.T) { ctx := context.Background() tr := newTranslator(t, zap.NewNop()) consumer := &mockFullConsumer{} - if err := tr.MapMetrics(ctx, createTestMetricWithAttributes(false, "process.runtime.dotnet.gc.heap.size", pmetric.MetricTypeGauge, "generation", []string{"gen0", "gen1", "gen2"}), consumer); err != nil { + attributes := []runtimeMetricAttribute{{ + key: "generation", + values: []string{"gen1"}, + }} + if err := tr.MapMetrics(ctx, createTestMetricWithAttributes(false, "process.runtime.dotnet.gc.heap.size", pmetric.MetricTypeGauge, attributes), consumer); err != nil { t.Fatal(err) } startTs := int(getProcessStartTime()) + 1 assert.ElementsMatch(t, consumer.metrics, []metric{ - newGaugeWithHost(newDims("process.runtime.dotnet.gc.heap.size").AddTags("generation:gen0"), uint64(seconds(startTs+1)), 10, fallbackHostname), - newGaugeWithHost(newDims("process.runtime.dotnet.gc.heap.size").AddTags("generation:gen1"), uint64(seconds(startTs+2)), 15, fallbackHostname), - newGaugeWithHost(newDims("process.runtime.dotnet.gc.heap.size").AddTags("generation:gen2"), uint64(seconds(startTs+3)), 20, fallbackHostname), - newGaugeWithHost(newDims("runtime.dotnet.gc.size.gen0").AddTags("generation:gen0"), uint64(seconds(startTs+1)), 10, fallbackHostname), - newGaugeWithHost(newDims("runtime.dotnet.gc.size.gen1").AddTags("generation:gen1"), uint64(seconds(startTs+2)), 15, fallbackHostname), - newGaugeWithHost(newDims("runtime.dotnet.gc.size.gen2").AddTags("generation:gen2"), uint64(seconds(startTs+3)), 20, fallbackHostname), + newGaugeWithHost(newDims("process.runtime.dotnet.gc.heap.size").AddTags("generation:gen1"), uint64(seconds(startTs+1)), 10, fallbackHostname), + newGaugeWithHost(newDims("runtime.dotnet.gc.size.gen1"), uint64(seconds(startTs+1)), 10, fallbackHostname), }, ) } -func TestMapGaugeRuntimeMetricWithInvalidAttributes(t *testing.T) { +func TestMapRuntimeMetricWithTwoAttributesHasMapping(t *testing.T) { ctx := context.Background() tr := newTranslator(t, zap.NewNop()) consumer := &mockFullConsumer{} - if err := tr.MapMetrics(ctx, createTestMetricWithAttributes(false, "process.runtime.jvm.memory.usage", pmetric.MetricTypeGauge, "type", []string{"heap", "heap1", "heap2", "non_heap"}), consumer); err != nil { + attributes := []runtimeMetricAttribute{{ + key: "pool", + values: []string{"g1_old_gen"}, + }, { + key: "type", + values: []string{"heap"}, + }} + if err := tr.MapMetrics(ctx, createTestMetricWithAttributes(false, "process.runtime.jvm.memory.usage", pmetric.MetricTypeGauge, attributes), consumer); err != nil { t.Fatal(err) } startTs := int(getProcessStartTime()) + 1 assert.ElementsMatch(t, consumer.metrics, []metric{ - newGaugeWithHost(newDims("process.runtime.jvm.memory.usage").AddTags("type:heap"), uint64(seconds(startTs+1)), 10, fallbackHostname), - newGaugeWithHost(newDims("process.runtime.jvm.memory.usage").AddTags("type:heap1"), uint64(seconds(startTs+2)), 15, fallbackHostname), - newGaugeWithHost(newDims("process.runtime.jvm.memory.usage").AddTags("type:heap2"), uint64(seconds(startTs+3)), 20, fallbackHostname), - newGaugeWithHost(newDims("process.runtime.jvm.memory.usage").AddTags("type:non_heap"), uint64(seconds(startTs+4)), 25, fallbackHostname), - newGaugeWithHost(newDims("jvm.heap_memory").AddTags("type:heap"), uint64(seconds(startTs+1)), 10, fallbackHostname), - newGaugeWithHost(newDims("jvm.non_heap_memory").AddTags("type:non_heap"), uint64(seconds(startTs+4)), 25, fallbackHostname), + newGaugeWithHost(newDims("process.runtime.jvm.memory.usage").AddTags("pool:g1_old_gen", "type:heap"), uint64(seconds(startTs+1)), 10, fallbackHostname), + newGaugeWithHost(newDims("jvm.heap_memory").AddTags("pool:g1_old_gen"), uint64(seconds(startTs+1)), 10, fallbackHostname), + newGaugeWithHost(newDims("jvm.gc.old_gen_size"), uint64(seconds(startTs+1)), 10, fallbackHostname), }, ) } +//func TestMapGaugeRuntimeMetricWithInvalidAttributes(t *testing.T) { +// ctx := context.Background() +// tr := newTranslator(t, zap.NewNop()) +// consumer := &mockFullConsumer{} +// if err := tr.MapMetrics(ctx, createTestMetricWithAttributes(false, "process.runtime.jvm.memory.usage", pmetric.MetricTypeGauge, "type", []string{"heap", "heap1", "heap2", "non_heap"}), consumer); err != nil { +// t.Fatal(err) +// } +// startTs := int(getProcessStartTime()) + 1 +// assert.ElementsMatch(t, +// consumer.metrics, +// []metric{ +// newGaugeWithHost(newDims("process.runtime.jvm.memory.usage").AddTags("type:heap"), uint64(seconds(startTs+1)), 10, fallbackHostname), +// newGaugeWithHost(newDims("process.runtime.jvm.memory.usage").AddTags("type:heap1"), uint64(seconds(startTs+2)), 15, fallbackHostname), +// newGaugeWithHost(newDims("process.runtime.jvm.memory.usage").AddTags("type:heap2"), uint64(seconds(startTs+3)), 20, fallbackHostname), +// newGaugeWithHost(newDims("process.runtime.jvm.memory.usage").AddTags("type:non_heap"), uint64(seconds(startTs+4)), 25, fallbackHostname), +// newGaugeWithHost(newDims("jvm.heap_memory").AddTags("type:heap"), uint64(seconds(startTs+1)), 10, fallbackHostname), +// newGaugeWithHost(newDims("jvm.non_heap_memory").AddTags("type:non_heap"), uint64(seconds(startTs+4)), 25, fallbackHostname), +// }, +// ) +//} + func TestMapRuntimeMetricsNoMapping(t *testing.T) { ctx := context.Background() tr := newTranslator(t, zap.NewNop()) @@ -847,7 +874,7 @@ func createTestDoubleCumulativeMonotonicMetrics(tsmatch bool) pmetric.Metrics { return md } -func createTestMetricWithAttributes(tsmatch bool, metricName string, metricType pmetric.MetricType, attribute string, attributeValues []string) pmetric.Metrics { +func createTestMetricWithAttributes(tsmatch bool, metricName string, metricType pmetric.MetricType, attributes []runtimeMetricAttribute) pmetric.Metrics { md := pmetric.NewMetrics() met := md.ResourceMetrics().AppendEmpty().ScopeMetrics().AppendEmpty().Metrics().AppendEmpty() met.SetName(metricName) @@ -862,23 +889,19 @@ func createTestMetricWithAttributes(tsmatch bool, metricName string, metricType dpsInt = met.Gauge().DataPoints() } - var values []int64 - for i := range attributeValues { - values = append(values, int64(10+(i*5))) - } - dpsInt.EnsureCapacity(len(values)) + dpsInt.EnsureCapacity(1) startTs := int(getProcessStartTime()) + 1 - for i, val := range values { - dpInt := dpsInt.AppendEmpty() - dpInt.Attributes().PutStr(attribute, attributeValues[i]) - dpInt.SetStartTimestamp(seconds(startTs)) - if tsmatch { - dpInt.SetTimestamp(seconds(startTs)) - } else { - dpInt.SetTimestamp(seconds(startTs + i + 1)) - } - dpInt.SetIntValue(val) + dpInt := dpsInt.AppendEmpty() + for _, attr := range attributes { + dpInt.Attributes().PutStr(attr.key, attr.values[0]) + } + dpInt.SetStartTimestamp(seconds(startTs)) + if tsmatch { + dpInt.SetTimestamp(seconds(startTs)) + } else { + dpInt.SetTimestamp(seconds(startTs + 1)) } + dpInt.SetIntValue(10) return md } From 553fc228c5e2af1e6910378cc9f13d883da67af7 Mon Sep 17 00:00:00 2001 From: Stanley Liu Date: Tue, 4 Apr 2023 14:04:43 -0400 Subject: [PATCH 10/22] Move runtime metric mappings to separate file --- pkg/otlp/metrics/metrics_translator.go | 163 ------------------- pkg/otlp/metrics/runtime_metric_mappings.go | 164 ++++++++++++++++++++ 2 files changed, 164 insertions(+), 163 deletions(-) create mode 100644 pkg/otlp/metrics/runtime_metric_mappings.go diff --git a/pkg/otlp/metrics/metrics_translator.go b/pkg/otlp/metrics/metrics_translator.go index 46220cd5..36481d00 100644 --- a/pkg/otlp/metrics/metrics_translator.go +++ b/pkg/otlp/metrics/metrics_translator.go @@ -34,169 +34,6 @@ import ( "github.com/DataDog/opentelemetry-mapping-go/pkg/quantile" ) -// runtimeMetricMapping defines the fields needed to map OTel runtime metrics to their equivalent -// Datadog runtime metrics -type runtimeMetricMapping struct { - mappedName string // the Datadog runtime metric name - attributes []runtimeMetricAttribute // the attribute(s) this metric originates from -} - -// runtimeMetricAttribute defines the structure for an attribute in regard to mapping runtime metrics. -// The presence of a runtimeMetricAttribute means that a metric must be mapped from a data point -// with the given attribute(s). -type runtimeMetricAttribute struct { - key string // the attribute name - values []string // the attribute value, or multiple values if there is more than one value for the same mapping -} - -var goRuntimeMetricsMappings = map[string][]runtimeMetricMapping{ - "process.runtime.go.goroutines": {{mappedName: "runtime.go.num_goroutine"}}, - "process.runtime.go.cgo.calls": {{mappedName: "runtime.go.num_cgo_call"}}, - "process.runtime.go.lookups": {{mappedName: "runtime.go.mem_stats.lookups"}}, - "process.runtime.go.mem.heap_alloc": {{mappedName: "runtime.go.mem_stats.heap_alloc"}}, - "process.runtime.go.mem.heap_sys": {{mappedName: "runtime.go.mem_stats.heap_sys"}}, - "process.runtime.go.mem.heap_idle": {{mappedName: "runtime.go.mem_stats.heap_idle"}}, - "process.runtime.go.mem.heap_inuse": {{mappedName: "runtime.go.mem_stats.heap_inuse"}}, - "process.runtime.go.mem.heap_released": {{mappedName: "runtime.go.mem_stats.heap_released"}}, - "process.runtime.go.mem.heap_objects": {{mappedName: "runtime.go.mem_stats.heap_objects"}}, - "process.runtime.go.gc.pause_total_ns": {{mappedName: "runtime.go.mem_stats.pause_total_ns"}}, - "process.runtime.go.gc.count": {{mappedName: "runtime.go.mem_stats.num_gc"}}, -} - -var dotnetRuntimeMetricsMappings = map[string][]runtimeMetricMapping{ - "process.runtime.dotnet.thread_pool.threads.count": {{mappedName: "runtime.dotnet.threads.count"}}, - "process.runtime.dotnet.monitor.lock_contention.count": {{mappedName: "runtime.dotnet.threads.contention_count"}}, - "process.runtime.dotnet.exceptions.count": {{mappedName: "runtime.dotnet.exceptions.count"}}, - "process.runtime.dotnet.gc.heap.size": {{ - mappedName: "runtime.dotnet.gc.size.gen0", - attributes: []runtimeMetricAttribute{{ - key: "generation", - values: []string{"gen0"}, - }}, - }, { - mappedName: "runtime.dotnet.gc.size.gen1", - attributes: []runtimeMetricAttribute{{ - key: "generation", - values: []string{"gen1"}, - }}, - }, { - mappedName: "runtime.dotnet.gc.size.gen2", - attributes: []runtimeMetricAttribute{{ - key: "generation", - values: []string{"gen2"}, - }}, - }, { - mappedName: "runtime.dotnet.gc.size.loh", - attributes: []runtimeMetricAttribute{{ - key: "generation", - values: []string{"loh"}, - }}, - }}, - "process.runtime.dotnet.gc.collections.count": {{ - mappedName: "runtime.dotnet.gc.count.gen0", - attributes: []runtimeMetricAttribute{{ - key: "generation", - values: []string{"gen0"}, - }}, - }, { - mappedName: "runtime.dotnet.gc.count.gen1", - attributes: []runtimeMetricAttribute{{ - key: "generation", - values: []string{"gen1"}, - }}, - }, { - mappedName: "runtime.dotnet.gc.count.gen2", - attributes: []runtimeMetricAttribute{{ - key: "generation", - values: []string{"gen2"}, - }}, - }}, -} - -var javaRuntimeMetricsMappings = map[string][]runtimeMetricMapping{ - "process.runtime.jvm.threads.count": {{mappedName: "jvm.thread_count"}}, - "process.runtime.jvm.gc.duration": {{mappedName: "jvm.gc.parnew.time"}}, - "process.runtime.jvm.memory.usage": {{ - mappedName: "jvm.heap_memory", - attributes: []runtimeMetricAttribute{{ - key: "type", - values: []string{"heap"}, - }}, - }, { - mappedName: "jvm.non_heap_memory", - attributes: []runtimeMetricAttribute{{ - key: "type", - values: []string{"non_heap"}, - }}, - }, { - mappedName: "jvm.gc.old_gen_size", - attributes: []runtimeMetricAttribute{{ - key: "pool", - values: []string{"g1_old_gen", "ps_old_gen", "tenured_gen"}, - }, { - key: "type", - values: []string{"heap"}, - }}, - }}, - "process.runtime.jvm.memory.committed": {{ - mappedName: "jvm.heap_memory_committed", - attributes: []runtimeMetricAttribute{{ - key: "type", - values: []string{"heap"}, - }}, - }, { - mappedName: "jvm.non_heap_memory_committed", - attributes: []runtimeMetricAttribute{{ - key: "type", - values: []string{"non_heap"}, - }}, - }}, - "process.runtime.jvm.memory.init": {{ - mappedName: "jvm.heap_memory_init", - attributes: []runtimeMetricAttribute{{ - key: "type", - values: []string{"heap"}, - }}, - }, { - mappedName: "jvm.non_heap_memory_init", - attributes: []runtimeMetricAttribute{{ - key: "type", - values: []string{"non_heap"}, - }}, - }}, - "process.runtime.jvm.memory.limit": {{ - mappedName: "jvm.heap_memory_max", - attributes: []runtimeMetricAttribute{{ - key: "type", - values: []string{"heap"}, - }}, - }, { - mappedName: "jvm.non_heap_memory_max", - attributes: []runtimeMetricAttribute{{ - key: "type", - values: []string{"non_heap"}, - }}, - }}, -} - -func getRuntimeMetricsMappings() map[string][]runtimeMetricMapping { - res := map[string][]runtimeMetricMapping{} - for k, v := range goRuntimeMetricsMappings { - res[k] = v - } - for k, v := range dotnetRuntimeMetricsMappings { - res[k] = v - } - for k, v := range javaRuntimeMetricsMappings { - res[k] = v - } - return res -} - -// runtimeMetricsMappings defines the mappings from OTel runtime metric names to their -// equivalent Datadog runtime metric names -var runtimeMetricsMappings = getRuntimeMetricsMappings() - const ( metricName string = "metric name" errNoBucketsNoSumCount string = "no buckets mode and no send count sum are incompatible" diff --git a/pkg/otlp/metrics/runtime_metric_mappings.go b/pkg/otlp/metrics/runtime_metric_mappings.go new file mode 100644 index 00000000..90a5ada4 --- /dev/null +++ b/pkg/otlp/metrics/runtime_metric_mappings.go @@ -0,0 +1,164 @@ +package metrics + +// runtimeMetricMapping defines the fields needed to map OTel runtime metrics to their equivalent +// Datadog runtime metrics +type runtimeMetricMapping struct { + mappedName string // the Datadog runtime metric name + attributes []runtimeMetricAttribute // the attribute(s) this metric originates from +} + +// runtimeMetricAttribute defines the structure for an attribute in regard to mapping runtime metrics. +// The presence of a runtimeMetricAttribute means that a metric must be mapped from a data point +// with the given attribute(s). +type runtimeMetricAttribute struct { + key string // the attribute name + values []string // the attribute value, or multiple values if there is more than one value for the same mapping +} + +var goRuntimeMetricsMappings = map[string][]runtimeMetricMapping{ + "process.runtime.go.goroutines": {{mappedName: "runtime.go.num_goroutine"}}, + "process.runtime.go.cgo.calls": {{mappedName: "runtime.go.num_cgo_call"}}, + "process.runtime.go.lookups": {{mappedName: "runtime.go.mem_stats.lookups"}}, + "process.runtime.go.mem.heap_alloc": {{mappedName: "runtime.go.mem_stats.heap_alloc"}}, + "process.runtime.go.mem.heap_sys": {{mappedName: "runtime.go.mem_stats.heap_sys"}}, + "process.runtime.go.mem.heap_idle": {{mappedName: "runtime.go.mem_stats.heap_idle"}}, + "process.runtime.go.mem.heap_inuse": {{mappedName: "runtime.go.mem_stats.heap_inuse"}}, + "process.runtime.go.mem.heap_released": {{mappedName: "runtime.go.mem_stats.heap_released"}}, + "process.runtime.go.mem.heap_objects": {{mappedName: "runtime.go.mem_stats.heap_objects"}}, + "process.runtime.go.gc.pause_total_ns": {{mappedName: "runtime.go.mem_stats.pause_total_ns"}}, + "process.runtime.go.gc.count": {{mappedName: "runtime.go.mem_stats.num_gc"}}, +} + +var dotnetRuntimeMetricsMappings = map[string][]runtimeMetricMapping{ + "process.runtime.dotnet.thread_pool.threads.count": {{mappedName: "runtime.dotnet.threads.count"}}, + "process.runtime.dotnet.monitor.lock_contention.count": {{mappedName: "runtime.dotnet.threads.contention_count"}}, + "process.runtime.dotnet.exceptions.count": {{mappedName: "runtime.dotnet.exceptions.count"}}, + "process.runtime.dotnet.gc.heap.size": {{ + mappedName: "runtime.dotnet.gc.size.gen0", + attributes: []runtimeMetricAttribute{{ + key: "generation", + values: []string{"gen0"}, + }}, + }, { + mappedName: "runtime.dotnet.gc.size.gen1", + attributes: []runtimeMetricAttribute{{ + key: "generation", + values: []string{"gen1"}, + }}, + }, { + mappedName: "runtime.dotnet.gc.size.gen2", + attributes: []runtimeMetricAttribute{{ + key: "generation", + values: []string{"gen2"}, + }}, + }, { + mappedName: "runtime.dotnet.gc.size.loh", + attributes: []runtimeMetricAttribute{{ + key: "generation", + values: []string{"loh"}, + }}, + }}, + "process.runtime.dotnet.gc.collections.count": {{ + mappedName: "runtime.dotnet.gc.count.gen0", + attributes: []runtimeMetricAttribute{{ + key: "generation", + values: []string{"gen0"}, + }}, + }, { + mappedName: "runtime.dotnet.gc.count.gen1", + attributes: []runtimeMetricAttribute{{ + key: "generation", + values: []string{"gen1"}, + }}, + }, { + mappedName: "runtime.dotnet.gc.count.gen2", + attributes: []runtimeMetricAttribute{{ + key: "generation", + values: []string{"gen2"}, + }}, + }}, +} + +var javaRuntimeMetricsMappings = map[string][]runtimeMetricMapping{ + "process.runtime.jvm.threads.count": {{mappedName: "jvm.thread_count"}}, + "process.runtime.jvm.gc.duration": {{mappedName: "jvm.gc.parnew.time"}}, + "process.runtime.jvm.memory.usage": {{ + mappedName: "jvm.heap_memory", + attributes: []runtimeMetricAttribute{{ + key: "type", + values: []string{"heap"}, + }}, + }, { + mappedName: "jvm.non_heap_memory", + attributes: []runtimeMetricAttribute{{ + key: "type", + values: []string{"non_heap"}, + }}, + }, { + mappedName: "jvm.gc.old_gen_size", + attributes: []runtimeMetricAttribute{{ + key: "pool", + values: []string{"g1_old_gen", "ps_old_gen", "tenured_gen"}, + }, { + key: "type", + values: []string{"heap"}, + }}, + }}, + "process.runtime.jvm.memory.committed": {{ + mappedName: "jvm.heap_memory_committed", + attributes: []runtimeMetricAttribute{{ + key: "type", + values: []string{"heap"}, + }}, + }, { + mappedName: "jvm.non_heap_memory_committed", + attributes: []runtimeMetricAttribute{{ + key: "type", + values: []string{"non_heap"}, + }}, + }}, + "process.runtime.jvm.memory.init": {{ + mappedName: "jvm.heap_memory_init", + attributes: []runtimeMetricAttribute{{ + key: "type", + values: []string{"heap"}, + }}, + }, { + mappedName: "jvm.non_heap_memory_init", + attributes: []runtimeMetricAttribute{{ + key: "type", + values: []string{"non_heap"}, + }}, + }}, + "process.runtime.jvm.memory.limit": {{ + mappedName: "jvm.heap_memory_max", + attributes: []runtimeMetricAttribute{{ + key: "type", + values: []string{"heap"}, + }}, + }, { + mappedName: "jvm.non_heap_memory_max", + attributes: []runtimeMetricAttribute{{ + key: "type", + values: []string{"non_heap"}, + }}, + }}, +} + +func getRuntimeMetricsMappings() map[string][]runtimeMetricMapping { + res := map[string][]runtimeMetricMapping{} + for k, v := range goRuntimeMetricsMappings { + res[k] = v + } + for k, v := range dotnetRuntimeMetricsMappings { + res[k] = v + } + for k, v := range javaRuntimeMetricsMappings { + res[k] = v + } + return res +} + +// runtimeMetricsMappings defines the mappings from OTel runtime metric names to their +// equivalent Datadog runtime metric names +var runtimeMetricsMappings = getRuntimeMetricsMappings() From 71814bc3f800dee09f21409448edc27b5345e256 Mon Sep 17 00:00:00 2001 From: Stanley Liu Date: Tue, 4 Apr 2023 14:05:43 -0400 Subject: [PATCH 11/22] Generate licenses --- LICENSE-3rdparty.csv | 26 ++++++++++++++------------ 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/LICENSE-3rdparty.csv b/LICENSE-3rdparty.csv index 4a4479f4..15770202 100644 --- a/LICENSE-3rdparty.csv +++ b/LICENSE-3rdparty.csv @@ -52,8 +52,8 @@ pkg/otlp/attributes,go.opentelemetry.io/collector/pdata/internal/data/protogen/r pkg/otlp/attributes,go.opentelemetry.io/collector/pdata/internal/data/protogen/trace/v1,Apache-2.0,Copyright The OpenTelemetry Authors pkg/otlp/attributes,go.opentelemetry.io/collector/pdata/pcommon,Apache-2.0,Copyright The OpenTelemetry Authors pkg/otlp/attributes,go.opentelemetry.io/collector/semconv/v1.6.1,Apache-2.0,Copyright The OpenTelemetry Authors -pkg/otlp/attributes,go.uber.org/atomic,MIT,"Copyright (c) 2016 Uber Technologies, Inc" -pkg/otlp/attributes,go.uber.org/multierr,MIT,"Copyright (c) 2017-2021 Uber Technologies, Inc" +pkg/otlp/attributes,go.uber.org/atomic,MIT,"Copyright (c) 2016 Uber Technologies, Inc | Copyright (c) 2016 Uber Technologies, Inc" +pkg/otlp/attributes,go.uber.org/multierr,MIT,"Copyright (c) 2017-2021 Uber Technologies, Inc | Copyright (c) 2017-2021 Uber Technologies, Inc" pkg/otlp/attributes,golang.org/x/net/http/httpguts,BSD-3-Clause,Copyright (c) 2009 The Go Authors. All rights reserved pkg/otlp/attributes,golang.org/x/net/http2,BSD-3-Clause,Copyright (c) 2009 The Go Authors. All rights reserved pkg/otlp/attributes,golang.org/x/net/http2/hpack,BSD-3-Clause,Copyright (c) 2009 The Go Authors. All rights reserved @@ -177,7 +177,7 @@ pkg/otlp/metrics,github.com/json-iterator/go,MIT,Copyright (c) 2016 json-iterato pkg/otlp/metrics,github.com/modern-go/concurrent,Apache-2.0,Copyright (c) 2018 Tao Wen pkg/otlp/metrics,github.com/modern-go/reflect2,Apache-2.0,Copyright (c) 2018 Tao Wen pkg/otlp/metrics,github.com/patrickmn/go-cache,MIT,"Copyright (c) 2012-2017 Patrick Mylund Nielsen and the go-cache contributors | This is a list of people who have contributed code to go-cache. They, or their | employers, are the copyright holders of the contributed code. Contributed code | is subject to the license restrictions listed in LICENSE (as they were when the | code was contributed.) | Dustin Sallings | Jason Mooberry | Sergey Shepelev | Alex Edwards " -pkg/otlp/metrics,github.com/philhofer/fwd,MIT,"Copyright (c) 2014-2015, Philip Hofer" +pkg/otlp/metrics,github.com/philhofer/fwd,MIT,"Copyright (c) 2014-2015, Philip Hofer | Copyright (c) 2014-2015, Philip Hofer" pkg/otlp/metrics,github.com/tinylib/msgp/msgp,MIT,Copyright (c) 2014 Philip Hofer | Copyright (c) 2009 The Go Authors (license at http://golang.org) where indicated pkg/otlp/metrics,go.opentelemetry.io/collector/pdata/internal,Apache-2.0,Copyright The OpenTelemetry Authors pkg/otlp/metrics,go.opentelemetry.io/collector/pdata/internal/data,Apache-2.0,Copyright The OpenTelemetry Authors @@ -195,15 +195,17 @@ pkg/otlp/metrics,go.opentelemetry.io/collector/pdata/pcommon,Apache-2.0,Copyrigh pkg/otlp/metrics,go.opentelemetry.io/collector/pdata/pmetric,Apache-2.0,Copyright The OpenTelemetry Authors pkg/otlp/metrics,go.opentelemetry.io/collector/pdata/pmetric/internal/pmetricjson,Apache-2.0,Copyright The OpenTelemetry Authors pkg/otlp/metrics,go.opentelemetry.io/collector/semconv/v1.6.1,Apache-2.0,Copyright The OpenTelemetry Authors -pkg/otlp/metrics,go.uber.org/atomic,MIT,"Copyright (c) 2016 Uber Technologies, Inc" -pkg/otlp/metrics,go.uber.org/multierr,MIT,"Copyright (c) 2017-2021 Uber Technologies, Inc" -pkg/otlp/metrics,go.uber.org/zap,MIT,"Copyright (c) 2016-2017 Uber Technologies, Inc" -pkg/otlp/metrics,go.uber.org/zap/buffer,MIT,"Copyright (c) 2016-2017 Uber Technologies, Inc" -pkg/otlp/metrics,go.uber.org/zap/internal,MIT,"Copyright (c) 2016-2017 Uber Technologies, Inc" -pkg/otlp/metrics,go.uber.org/zap/internal/bufferpool,MIT,"Copyright (c) 2016-2017 Uber Technologies, Inc" -pkg/otlp/metrics,go.uber.org/zap/internal/color,MIT,"Copyright (c) 2016-2017 Uber Technologies, Inc" -pkg/otlp/metrics,go.uber.org/zap/internal/exit,MIT,"Copyright (c) 2016-2017 Uber Technologies, Inc" -pkg/otlp/metrics,go.uber.org/zap/zapcore,MIT,"Copyright (c) 2016-2017 Uber Technologies, Inc" +pkg/otlp/metrics,go.uber.org/atomic,MIT,"Copyright (c) 2016 Uber Technologies, Inc | Copyright (c) 2016 Uber Technologies, Inc" +pkg/otlp/metrics,go.uber.org/multierr,MIT,"Copyright (c) 2017-2021 Uber Technologies, Inc | Copyright (c) 2017-2021 Uber Technologies, Inc" +pkg/otlp/metrics,go.uber.org/zap,MIT,"Copyright (c) 2016-2017 Uber Technologies, Inc | Copyright (c) 2016-2017 Uber Technologies, Inc" +pkg/otlp/metrics,go.uber.org/zap/buffer,MIT,"Copyright (c) 2016-2017 Uber Technologies, Inc | Copyright (c) 2016-2017 Uber Technologies, Inc" +pkg/otlp/metrics,go.uber.org/zap/internal,MIT,"Copyright (c) 2016-2017 Uber Technologies, Inc | Copyright (c) 2016-2017 Uber Technologies, Inc" +pkg/otlp/metrics,go.uber.org/zap/internal/bufferpool,MIT,"Copyright (c) 2016-2017 Uber Technologies, Inc | Copyright (c) 2016-2017 Uber Technologies, Inc" +pkg/otlp/metrics,go.uber.org/zap/internal/color,MIT,"Copyright (c) 2016-2017 Uber Technologies, Inc | Copyright (c) 2016-2017 Uber Technologies, Inc" +pkg/otlp/metrics,go.uber.org/zap/internal/exit,MIT,"Copyright (c) 2016-2017 Uber Technologies, Inc | Copyright (c) 2016-2017 Uber Technologies, Inc" +pkg/otlp/metrics,go.uber.org/zap/zapcore,MIT,"Copyright (c) 2016-2017 Uber Technologies, Inc | Copyright (c) 2016-2017 Uber Technologies, Inc" +pkg/otlp/metrics,golang.org/x/exp/constraints,BSD-3-Clause,Copyright (c) 2009 The Go Authors. All rights reserved +pkg/otlp/metrics,golang.org/x/exp/slices,BSD-3-Clause,Copyright (c) 2009 The Go Authors. All rights reserved pkg/otlp/metrics,golang.org/x/net/http/httpguts,BSD-3-Clause,Copyright (c) 2009 The Go Authors. All rights reserved pkg/otlp/metrics,golang.org/x/net/http2,BSD-3-Clause,Copyright (c) 2009 The Go Authors. All rights reserved pkg/otlp/metrics,golang.org/x/net/http2/hpack,BSD-3-Clause,Copyright (c) 2009 The Go Authors. All rights reserved From 58a4e4d568a09831a975e840396ec746955be48f Mon Sep 17 00:00:00 2001 From: Stanley Liu Date: Tue, 4 Apr 2023 15:36:32 -0400 Subject: [PATCH 12/22] Support mapping histograms, add rest of jvm mappings --- pkg/otlp/metrics/metrics_translator.go | 33 ++++ pkg/otlp/metrics/metrics_translator_test.go | 160 ++++++++++++++++---- pkg/otlp/metrics/runtime_metric_mappings.go | 75 ++++++++- 3 files changed, 238 insertions(+), 30 deletions(-) diff --git a/pkg/otlp/metrics/metrics_translator.go b/pkg/otlp/metrics/metrics_translator.go index 36481d00..b1415a6e 100644 --- a/pkg/otlp/metrics/metrics_translator.go +++ b/pkg/otlp/metrics/metrics_translator.go @@ -512,6 +512,37 @@ func mapSumRuntimeMetricWithAttributes(md pmetric.Metric, metricsArray pmetric.M } } +// mapHistogramRuntimeMetricWithAttributes maps the specified runtime metric from metric attributes into a new Histogram metric +func mapHistogramRuntimeMetricWithAttributes(md pmetric.Metric, metricsArray pmetric.MetricSlice, mp runtimeMetricMapping) { + for i := 0; i < md.Histogram().DataPoints().Len(); i++ { + matchesAttributes := true + for _, attribute := range mp.attributes { + attributeValue, res := md.Histogram().DataPoints().At(i).Attributes().Get(attribute.key) + if !res || !slices.Contains(attribute.values, attributeValue.AsString()) { + matchesAttributes = false + break + } + } + if matchesAttributes { + cp := metricsArray.AppendEmpty() + cp.SetEmptyHistogram() + cp.Histogram().SetAggregationTemporality(md.Histogram().AggregationTemporality()) + dataPoint := cp.Histogram().DataPoints().AppendEmpty() + md.Histogram().DataPoints().At(i).CopyTo(dataPoint) + dataPoint.Attributes().RemoveIf(func(s string, value pcommon.Value) bool { + for _, attribute := range mp.attributes { + if s == attribute.key { + return true + } + } + return false + }) + cp.SetName(mp.mappedName) + break + } + } +} + // MapMetrics maps OTLP metrics into the DataDog format func (t *Translator) MapMetrics(ctx context.Context, md pmetric.Metrics, consumer Consumer) error { rms := md.ResourceMetrics() @@ -574,6 +605,8 @@ func (t *Translator) MapMetrics(ctx context.Context, md pmetric.Metrics, consume mapSumRuntimeMetricWithAttributes(md, metricsArray, mp) } else if md.Type() == pmetric.MetricTypeGauge { mapGaugeRuntimeMetricWithAttributes(md, metricsArray, mp) + } else if md.Type() == pmetric.MetricTypeHistogram { + mapHistogramRuntimeMetricWithAttributes(md, metricsArray, mp) } } } diff --git a/pkg/otlp/metrics/metrics_translator_test.go b/pkg/otlp/metrics/metrics_translator_test.go index d0d47b10..3faf2376 100644 --- a/pkg/otlp/metrics/metrics_translator_test.go +++ b/pkg/otlp/metrics/metrics_translator_test.go @@ -16,6 +16,7 @@ package metrics import ( "context" + "fmt" "math" "testing" "time" @@ -102,6 +103,7 @@ func newTranslator(t *testing.T, logger *zap.Logger) *Translator { WithFallbackSourceProvider(testProvider(fallbackHostname)), WithHistogramMode(HistogramModeDistributions), WithNumberMode(NumberModeCumulativeToDelta), + WithHistogramAggregations(), } tr, err := NewTranslator( @@ -465,6 +467,58 @@ func TestMapGaugeRuntimeMetricWithAttributesHasMapping(t *testing.T) { ) } +func TestMapHistogramRuntimeMetricHasMapping(t *testing.T) { + ctx := context.Background() + tr := newTranslator(t, zap.NewNop()) + consumer := &mockFullConsumer{} + fmt.Println(consumer.metrics) + + if err := tr.MapMetrics(ctx, createTestHistogramMetric(false, "process.runtime.jvm.gc.duration"), consumer); err != nil { + t.Fatal(err) + } + startTs := int(getProcessStartTime()) + 1 + assert.ElementsMatch(t, + consumer.metrics, + []metric{ + newCountWithHost(newDims("process.runtime.jvm.gc.duration.count"), uint64(seconds(startTs+1)), 100, fallbackHostname), + newCountWithHost(newDims("process.runtime.jvm.gc.duration.sum"), uint64(seconds(startTs+1)), 0, fallbackHostname), + newGaugeWithHost(newDims("process.runtime.jvm.gc.duration.min"), uint64(seconds(startTs+1)), -100, fallbackHostname), + newGaugeWithHost(newDims("process.runtime.jvm.gc.duration.max"), uint64(seconds(startTs+1)), 100, fallbackHostname), + newCountWithHost(newDims("jvm.gc.parnew.time.count"), uint64(seconds(startTs+1)), 100, fallbackHostname), + newCountWithHost(newDims("jvm.gc.parnew.time.sum"), uint64(seconds(startTs+1)), 0, fallbackHostname), + newGaugeWithHost(newDims("jvm.gc.parnew.time.min"), uint64(seconds(startTs+1)), -100, fallbackHostname), + newGaugeWithHost(newDims("jvm.gc.parnew.time.max"), uint64(seconds(startTs+1)), 100, fallbackHostname), + }, + ) +} + +func TestMapHistogramRuntimeMetricWithAttributesHasMapping(t *testing.T) { + ctx := context.Background() + tr := newTranslator(t, zap.NewNop()) + consumer := &mockFullConsumer{} + attributes := []runtimeMetricAttribute{{ + key: "generation", + values: []string{"gen1"}, + }} + if err := tr.MapMetrics(ctx, createTestMetricWithAttributes(false, "process.runtime.dotnet.gc.heap.size", pmetric.MetricTypeHistogram, attributes), consumer); err != nil { + t.Fatal(err) + } + startTs := int(getProcessStartTime()) + 1 + assert.ElementsMatch(t, + consumer.metrics, + []metric{ + newCountWithHost(newDims("process.runtime.dotnet.gc.heap.size.count").AddTags("generation:gen1"), uint64(seconds(startTs+1)), 100, fallbackHostname), + newCountWithHost(newDims("process.runtime.dotnet.gc.heap.size.sum").AddTags("generation:gen1"), uint64(seconds(startTs+1)), 0, fallbackHostname), + newGaugeWithHost(newDims("process.runtime.dotnet.gc.heap.size.min").AddTags("generation:gen1"), uint64(seconds(startTs+1)), -100, fallbackHostname), + newGaugeWithHost(newDims("process.runtime.dotnet.gc.heap.size.max").AddTags("generation:gen1"), uint64(seconds(startTs+1)), 100, fallbackHostname), + newCountWithHost(newDims("runtime.dotnet.gc.size.gen1.count"), uint64(seconds(startTs+1)), 100, fallbackHostname), + newCountWithHost(newDims("runtime.dotnet.gc.size.gen1.sum"), uint64(seconds(startTs+1)), 0, fallbackHostname), + newGaugeWithHost(newDims("runtime.dotnet.gc.size.gen1.min"), uint64(seconds(startTs+1)), -100, fallbackHostname), + newGaugeWithHost(newDims("runtime.dotnet.gc.size.gen1.max"), uint64(seconds(startTs+1)), 100, fallbackHostname), + }, + ) +} + func TestMapRuntimeMetricWithTwoAttributesHasMapping(t *testing.T) { ctx := context.Background() tr := newTranslator(t, zap.NewNop()) @@ -490,26 +544,25 @@ func TestMapRuntimeMetricWithTwoAttributesHasMapping(t *testing.T) { ) } -//func TestMapGaugeRuntimeMetricWithInvalidAttributes(t *testing.T) { -// ctx := context.Background() -// tr := newTranslator(t, zap.NewNop()) -// consumer := &mockFullConsumer{} -// if err := tr.MapMetrics(ctx, createTestMetricWithAttributes(false, "process.runtime.jvm.memory.usage", pmetric.MetricTypeGauge, "type", []string{"heap", "heap1", "heap2", "non_heap"}), consumer); err != nil { -// t.Fatal(err) -// } -// startTs := int(getProcessStartTime()) + 1 -// assert.ElementsMatch(t, -// consumer.metrics, -// []metric{ -// newGaugeWithHost(newDims("process.runtime.jvm.memory.usage").AddTags("type:heap"), uint64(seconds(startTs+1)), 10, fallbackHostname), -// newGaugeWithHost(newDims("process.runtime.jvm.memory.usage").AddTags("type:heap1"), uint64(seconds(startTs+2)), 15, fallbackHostname), -// newGaugeWithHost(newDims("process.runtime.jvm.memory.usage").AddTags("type:heap2"), uint64(seconds(startTs+3)), 20, fallbackHostname), -// newGaugeWithHost(newDims("process.runtime.jvm.memory.usage").AddTags("type:non_heap"), uint64(seconds(startTs+4)), 25, fallbackHostname), -// newGaugeWithHost(newDims("jvm.heap_memory").AddTags("type:heap"), uint64(seconds(startTs+1)), 10, fallbackHostname), -// newGaugeWithHost(newDims("jvm.non_heap_memory").AddTags("type:non_heap"), uint64(seconds(startTs+4)), 25, fallbackHostname), -// }, -// ) -//} +func TestMapGaugeRuntimeMetricWithInvalidAttributes(t *testing.T) { + ctx := context.Background() + tr := newTranslator(t, zap.NewNop()) + consumer := &mockFullConsumer{} + attributes := []runtimeMetricAttribute{{ + key: "type", + values: []string{"heap2"}, + }} + if err := tr.MapMetrics(ctx, createTestMetricWithAttributes(false, "process.runtime.jvm.memory.usage", pmetric.MetricTypeGauge, attributes), consumer); err != nil { + t.Fatal(err) + } + startTs := int(getProcessStartTime()) + 1 + assert.ElementsMatch(t, + consumer.metrics, + []metric{ + newGaugeWithHost(newDims("process.runtime.jvm.memory.usage").AddTags("type:heap2"), uint64(seconds(startTs+1)), 10, fallbackHostname), + }, + ) +} func TestMapRuntimeMetricsNoMapping(t *testing.T) { ctx := context.Background() @@ -874,11 +927,38 @@ func createTestDoubleCumulativeMonotonicMetrics(tsmatch bool) pmetric.Metrics { return md } +func createTestHistogramMetric(tsmatch bool, metricName string) pmetric.Metrics { + md := pmetric.NewMetrics() + met := md.ResourceMetrics().AppendEmpty().ScopeMetrics().AppendEmpty().Metrics().AppendEmpty() + met.SetName(metricName) + var hpsCount pmetric.HistogramDataPointSlice + met.SetEmptyHistogram() + met.Histogram().SetAggregationTemporality(pmetric.AggregationTemporalityDelta) + hpsCount = met.Histogram().DataPoints() + hpsCount.EnsureCapacity(1) + startTs := int(getProcessStartTime()) + 1 + hpCount := hpsCount.AppendEmpty() + hpCount.SetStartTimestamp(seconds(startTs)) + if tsmatch { + hpCount.SetTimestamp(seconds(startTs)) + } else { + hpCount.SetTimestamp(seconds(startTs + 1)) + } + hpCount.ExplicitBounds().FromRaw([]float64{}) + hpCount.BucketCounts().FromRaw([]uint64{100}) + hpCount.SetCount(100) + hpCount.SetSum(0) + hpCount.SetMin(-100) + hpCount.SetMax(100) + return md +} + func createTestMetricWithAttributes(tsmatch bool, metricName string, metricType pmetric.MetricType, attributes []runtimeMetricAttribute) pmetric.Metrics { md := pmetric.NewMetrics() met := md.ResourceMetrics().AppendEmpty().ScopeMetrics().AppendEmpty().Metrics().AppendEmpty() met.SetName(metricName) var dpsInt pmetric.NumberDataPointSlice + var hpsCount pmetric.HistogramDataPointSlice if metricType == pmetric.MetricTypeSum { met.SetEmptySum() met.Sum().SetAggregationTemporality(pmetric.AggregationTemporalityCumulative) @@ -887,21 +967,47 @@ func createTestMetricWithAttributes(tsmatch bool, metricName string, metricType } else if metricType == pmetric.MetricTypeGauge { met.SetEmptyGauge() dpsInt = met.Gauge().DataPoints() + } else if metricType == pmetric.MetricTypeHistogram { + met.SetEmptyHistogram() + met.Histogram().SetAggregationTemporality(pmetric.AggregationTemporalityDelta) + hpsCount = met.Histogram().DataPoints() + } + + if metricType != pmetric.MetricTypeHistogram { + dpsInt.EnsureCapacity(1) + startTs := int(getProcessStartTime()) + 1 + dpInt := dpsInt.AppendEmpty() + for _, attr := range attributes { + dpInt.Attributes().PutStr(attr.key, attr.values[0]) + } + dpInt.SetStartTimestamp(seconds(startTs)) + if tsmatch { + dpInt.SetTimestamp(seconds(startTs)) + } else { + dpInt.SetTimestamp(seconds(startTs + 1)) + } + dpInt.SetIntValue(10) + return md } - dpsInt.EnsureCapacity(1) + hpsCount.EnsureCapacity(1) startTs := int(getProcessStartTime()) + 1 - dpInt := dpsInt.AppendEmpty() + hpCount := hpsCount.AppendEmpty() for _, attr := range attributes { - dpInt.Attributes().PutStr(attr.key, attr.values[0]) + hpCount.Attributes().PutStr(attr.key, attr.values[0]) } - dpInt.SetStartTimestamp(seconds(startTs)) + hpCount.SetStartTimestamp(seconds(startTs)) if tsmatch { - dpInt.SetTimestamp(seconds(startTs)) + hpCount.SetTimestamp(seconds(startTs)) } else { - dpInt.SetTimestamp(seconds(startTs + 1)) + hpCount.SetTimestamp(seconds(startTs + 1)) } - dpInt.SetIntValue(10) + hpCount.ExplicitBounds().FromRaw([]float64{}) + hpCount.BucketCounts().FromRaw([]uint64{100}) + hpCount.SetCount(100) + hpCount.SetSum(0) + hpCount.SetMin(-100) + hpCount.SetMax(100) return md } diff --git a/pkg/otlp/metrics/runtime_metric_mappings.go b/pkg/otlp/metrics/runtime_metric_mappings.go index 90a5ada4..15b8d92b 100644 --- a/pkg/otlp/metrics/runtime_metric_mappings.go +++ b/pkg/otlp/metrics/runtime_metric_mappings.go @@ -80,8 +80,11 @@ var dotnetRuntimeMetricsMappings = map[string][]runtimeMetricMapping{ } var javaRuntimeMetricsMappings = map[string][]runtimeMetricMapping{ - "process.runtime.jvm.threads.count": {{mappedName: "jvm.thread_count"}}, - "process.runtime.jvm.gc.duration": {{mappedName: "jvm.gc.parnew.time"}}, + "process.runtime.jvm.threads.count": {{mappedName: "jvm.thread_count"}}, + "process.runtime.jvm.classes.loaded": {{mappedName: "jvm.loaded_classes"}}, + "process.runtime.jvm.system.cpu.utilization": {{mappedName: "jvm.cpu_load.system"}}, + "process.runtime.jvm.cpu.utilization": {{mappedName: "jvm.cpu_load.process"}}, + "process.runtime.jvm.gc.duration": {{mappedName: "jvm.gc.parnew.time"}}, "process.runtime.jvm.memory.usage": {{ mappedName: "jvm.heap_memory", attributes: []runtimeMetricAttribute{{ @@ -98,11 +101,38 @@ var javaRuntimeMetricsMappings = map[string][]runtimeMetricMapping{ mappedName: "jvm.gc.old_gen_size", attributes: []runtimeMetricAttribute{{ key: "pool", - values: []string{"g1_old_gen", "ps_old_gen", "tenured_gen"}, + values: []string{"g1_old_gen", "tenured_gen", "ps_old_gen"}, }, { key: "type", values: []string{"heap"}, }}, + }, { + mappedName: "jvm.gc.eden_size", + attributes: []runtimeMetricAttribute{{ + key: "pool", + values: []string{"g1_eden_space", "eden_space", "par_eden_space", "ps_eden_space"}, + }, { + key: "type", + values: []string{"heap"}, + }}, + }, { + mappedName: "jvm.gc.survivor_size", + attributes: []runtimeMetricAttribute{{ + key: "pool", + values: []string{"g1_survivor_space", "survivor_space", "par_survivor_space", "ps_survivor_space"}, + }, { + key: "type", + values: []string{"heap"}, + }}, + }, { + mappedName: "jvm.gc.metaspace_size", + attributes: []runtimeMetricAttribute{{ + key: "pool", + values: []string{"metaspace"}, + }, { + key: "type", + values: []string{"non_heap"}, + }}, }}, "process.runtime.jvm.memory.committed": {{ mappedName: "jvm.heap_memory_committed", @@ -143,6 +173,45 @@ var javaRuntimeMetricsMappings = map[string][]runtimeMetricMapping{ values: []string{"non_heap"}, }}, }}, + "process.runtime.jvm.buffer.usage": {{ + mappedName: "jvm.buffer_pool.direct.used", + attributes: []runtimeMetricAttribute{{ + key: "pool", + values: []string{"direct"}, + }}, + }, { + mappedName: "jvm.buffer_pool.mapped.used", + attributes: []runtimeMetricAttribute{{ + key: "pool", + values: []string{"mapped"}, + }}, + }}, + "process.runtime.jvm.buffer.count": {{ + mappedName: "jvm.buffer_pool.direct.count", + attributes: []runtimeMetricAttribute{{ + key: "pool", + values: []string{"direct"}, + }}, + }, { + mappedName: "jvm.buffer_pool.mapped.count", + attributes: []runtimeMetricAttribute{{ + key: "pool", + values: []string{"mapped"}, + }}, + }}, + "process.runtime.jvm.buffer.limit": {{ + mappedName: "jvm.buffer_pool.direct.limit", + attributes: []runtimeMetricAttribute{{ + key: "pool", + values: []string{"direct"}, + }}, + }, { + mappedName: "jvm.buffer_pool.mapped.limit", + attributes: []runtimeMetricAttribute{{ + key: "pool", + values: []string{"mapped"}, + }}, + }}, } func getRuntimeMetricsMappings() map[string][]runtimeMetricMapping { From 5127316b8e7e98178a4babe1de29c6cc60393c31 Mon Sep 17 00:00:00 2001 From: Stanley Liu Date: Tue, 4 Apr 2023 17:53:37 -0400 Subject: [PATCH 13/22] Generate licenses --- LICENSE-3rdparty.csv | 24 +++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/LICENSE-3rdparty.csv b/LICENSE-3rdparty.csv index 8e1f0689..ce7584c0 100644 --- a/LICENSE-3rdparty.csv +++ b/LICENSE-3rdparty.csv @@ -52,7 +52,7 @@ pkg/otlp/attributes,go.opentelemetry.io/collector/pdata/internal/data/protogen/r pkg/otlp/attributes,go.opentelemetry.io/collector/pdata/internal/data/protogen/trace/v1,Apache-2.0,Copyright The OpenTelemetry Authors pkg/otlp/attributes,go.opentelemetry.io/collector/pdata/pcommon,Apache-2.0,Copyright The OpenTelemetry Authors pkg/otlp/attributes,go.opentelemetry.io/collector/semconv/v1.6.1,Apache-2.0,Copyright The OpenTelemetry Authors -pkg/otlp/attributes,go.uber.org/multierr,MIT,"Copyright (c) 2017-2021 Uber Technologies, Inc" +pkg/otlp/attributes,go.uber.org/multierr,MIT,"Copyright (c) 2017-2021 Uber Technologies, Inc | Copyright (c) 2017-2021 Uber Technologies, Inc" pkg/otlp/attributes,golang.org/x/net/http/httpguts,BSD-3-Clause,Copyright (c) 2009 The Go Authors. All rights reserved pkg/otlp/attributes,golang.org/x/net/http2,BSD-3-Clause,Copyright (c) 2009 The Go Authors. All rights reserved pkg/otlp/attributes,golang.org/x/net/http2/hpack,BSD-3-Clause,Copyright (c) 2009 The Go Authors. All rights reserved @@ -176,7 +176,7 @@ pkg/otlp/metrics,github.com/json-iterator/go,MIT,Copyright (c) 2016 json-iterato pkg/otlp/metrics,github.com/modern-go/concurrent,Apache-2.0,Copyright (c) 2018 Tao Wen pkg/otlp/metrics,github.com/modern-go/reflect2,Apache-2.0,Copyright (c) 2018 Tao Wen pkg/otlp/metrics,github.com/patrickmn/go-cache,MIT,"Copyright (c) 2012-2017 Patrick Mylund Nielsen and the go-cache contributors | This is a list of people who have contributed code to go-cache. They, or their | employers, are the copyright holders of the contributed code. Contributed code | is subject to the license restrictions listed in LICENSE (as they were when the | code was contributed.) | Dustin Sallings | Jason Mooberry | Sergey Shepelev | Alex Edwards " -pkg/otlp/metrics,github.com/philhofer/fwd,MIT,"Copyright (c) 2014-2015, Philip Hofer" +pkg/otlp/metrics,github.com/philhofer/fwd,MIT,"Copyright (c) 2014-2015, Philip Hofer | Copyright (c) 2014-2015, Philip Hofer" pkg/otlp/metrics,github.com/tinylib/msgp/msgp,MIT,Copyright (c) 2014 Philip Hofer | Copyright (c) 2009 The Go Authors (license at http://golang.org) where indicated pkg/otlp/metrics,go.opentelemetry.io/collector/pdata/internal,Apache-2.0,Copyright The OpenTelemetry Authors pkg/otlp/metrics,go.opentelemetry.io/collector/pdata/internal/data,Apache-2.0,Copyright The OpenTelemetry Authors @@ -194,15 +194,17 @@ pkg/otlp/metrics,go.opentelemetry.io/collector/pdata/pcommon,Apache-2.0,Copyrigh pkg/otlp/metrics,go.opentelemetry.io/collector/pdata/pmetric,Apache-2.0,Copyright The OpenTelemetry Authors pkg/otlp/metrics,go.opentelemetry.io/collector/pdata/pmetric/internal/pmetricjson,Apache-2.0,Copyright The OpenTelemetry Authors pkg/otlp/metrics,go.opentelemetry.io/collector/semconv/v1.6.1,Apache-2.0,Copyright The OpenTelemetry Authors -pkg/otlp/metrics,go.uber.org/atomic,MIT,"Copyright (c) 2016 Uber Technologies, Inc" -pkg/otlp/metrics,go.uber.org/multierr,MIT,"Copyright (c) 2017-2021 Uber Technologies, Inc" -pkg/otlp/metrics,go.uber.org/zap,MIT,"Copyright (c) 2016-2017 Uber Technologies, Inc" -pkg/otlp/metrics,go.uber.org/zap/buffer,MIT,"Copyright (c) 2016-2017 Uber Technologies, Inc" -pkg/otlp/metrics,go.uber.org/zap/internal,MIT,"Copyright (c) 2016-2017 Uber Technologies, Inc" -pkg/otlp/metrics,go.uber.org/zap/internal/bufferpool,MIT,"Copyright (c) 2016-2017 Uber Technologies, Inc" -pkg/otlp/metrics,go.uber.org/zap/internal/color,MIT,"Copyright (c) 2016-2017 Uber Technologies, Inc" -pkg/otlp/metrics,go.uber.org/zap/internal/exit,MIT,"Copyright (c) 2016-2017 Uber Technologies, Inc" -pkg/otlp/metrics,go.uber.org/zap/zapcore,MIT,"Copyright (c) 2016-2017 Uber Technologies, Inc" +pkg/otlp/metrics,go.uber.org/atomic,MIT,"Copyright (c) 2016 Uber Technologies, Inc | Copyright (c) 2016 Uber Technologies, Inc" +pkg/otlp/metrics,go.uber.org/multierr,MIT,"Copyright (c) 2017-2021 Uber Technologies, Inc | Copyright (c) 2017-2021 Uber Technologies, Inc" +pkg/otlp/metrics,go.uber.org/zap,MIT,"Copyright (c) 2016-2017 Uber Technologies, Inc | Copyright (c) 2016-2017 Uber Technologies, Inc" +pkg/otlp/metrics,go.uber.org/zap/buffer,MIT,"Copyright (c) 2016-2017 Uber Technologies, Inc | Copyright (c) 2016-2017 Uber Technologies, Inc" +pkg/otlp/metrics,go.uber.org/zap/internal,MIT,"Copyright (c) 2016-2017 Uber Technologies, Inc | Copyright (c) 2016-2017 Uber Technologies, Inc" +pkg/otlp/metrics,go.uber.org/zap/internal/bufferpool,MIT,"Copyright (c) 2016-2017 Uber Technologies, Inc | Copyright (c) 2016-2017 Uber Technologies, Inc" +pkg/otlp/metrics,go.uber.org/zap/internal/color,MIT,"Copyright (c) 2016-2017 Uber Technologies, Inc | Copyright (c) 2016-2017 Uber Technologies, Inc" +pkg/otlp/metrics,go.uber.org/zap/internal/exit,MIT,"Copyright (c) 2016-2017 Uber Technologies, Inc | Copyright (c) 2016-2017 Uber Technologies, Inc" +pkg/otlp/metrics,go.uber.org/zap/zapcore,MIT,"Copyright (c) 2016-2017 Uber Technologies, Inc | Copyright (c) 2016-2017 Uber Technologies, Inc" +pkg/otlp/metrics,golang.org/x/exp/constraints,BSD-3-Clause,Copyright (c) 2009 The Go Authors. All rights reserved +pkg/otlp/metrics,golang.org/x/exp/slices,BSD-3-Clause,Copyright (c) 2009 The Go Authors. All rights reserved pkg/otlp/metrics,golang.org/x/net/http/httpguts,BSD-3-Clause,Copyright (c) 2009 The Go Authors. All rights reserved pkg/otlp/metrics,golang.org/x/net/http2,BSD-3-Clause,Copyright (c) 2009 The Go Authors. All rights reserved pkg/otlp/metrics,golang.org/x/net/http2/hpack,BSD-3-Clause,Copyright (c) 2009 The Go Authors. All rights reserved From 5e3cce8d00ecb721c1e0a1242763ee29f4b50c50 Mon Sep 17 00:00:00 2001 From: Stanley Liu Date: Wed, 5 Apr 2023 12:48:34 -0400 Subject: [PATCH 14/22] License patch --- LICENSE-3rdparty.csv | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/LICENSE-3rdparty.csv b/LICENSE-3rdparty.csv index ce7584c0..7c4f123d 100644 --- a/LICENSE-3rdparty.csv +++ b/LICENSE-3rdparty.csv @@ -52,7 +52,7 @@ pkg/otlp/attributes,go.opentelemetry.io/collector/pdata/internal/data/protogen/r pkg/otlp/attributes,go.opentelemetry.io/collector/pdata/internal/data/protogen/trace/v1,Apache-2.0,Copyright The OpenTelemetry Authors pkg/otlp/attributes,go.opentelemetry.io/collector/pdata/pcommon,Apache-2.0,Copyright The OpenTelemetry Authors pkg/otlp/attributes,go.opentelemetry.io/collector/semconv/v1.6.1,Apache-2.0,Copyright The OpenTelemetry Authors -pkg/otlp/attributes,go.uber.org/multierr,MIT,"Copyright (c) 2017-2021 Uber Technologies, Inc | Copyright (c) 2017-2021 Uber Technologies, Inc" +pkg/otlp/attributes,go.uber.org/multierr,MIT,"Copyright (c) 2017-2021 Uber Technologies, Inc" pkg/otlp/attributes,golang.org/x/net/http/httpguts,BSD-3-Clause,Copyright (c) 2009 The Go Authors. All rights reserved pkg/otlp/attributes,golang.org/x/net/http2,BSD-3-Clause,Copyright (c) 2009 The Go Authors. All rights reserved pkg/otlp/attributes,golang.org/x/net/http2/hpack,BSD-3-Clause,Copyright (c) 2009 The Go Authors. All rights reserved @@ -176,7 +176,7 @@ pkg/otlp/metrics,github.com/json-iterator/go,MIT,Copyright (c) 2016 json-iterato pkg/otlp/metrics,github.com/modern-go/concurrent,Apache-2.0,Copyright (c) 2018 Tao Wen pkg/otlp/metrics,github.com/modern-go/reflect2,Apache-2.0,Copyright (c) 2018 Tao Wen pkg/otlp/metrics,github.com/patrickmn/go-cache,MIT,"Copyright (c) 2012-2017 Patrick Mylund Nielsen and the go-cache contributors | This is a list of people who have contributed code to go-cache. They, or their | employers, are the copyright holders of the contributed code. Contributed code | is subject to the license restrictions listed in LICENSE (as they were when the | code was contributed.) | Dustin Sallings | Jason Mooberry | Sergey Shepelev | Alex Edwards " -pkg/otlp/metrics,github.com/philhofer/fwd,MIT,"Copyright (c) 2014-2015, Philip Hofer | Copyright (c) 2014-2015, Philip Hofer" +pkg/otlp/metrics,github.com/philhofer/fwd,MIT,"Copyright (c) 2014-2015, Philip Hofer" pkg/otlp/metrics,github.com/tinylib/msgp/msgp,MIT,Copyright (c) 2014 Philip Hofer | Copyright (c) 2009 The Go Authors (license at http://golang.org) where indicated pkg/otlp/metrics,go.opentelemetry.io/collector/pdata/internal,Apache-2.0,Copyright The OpenTelemetry Authors pkg/otlp/metrics,go.opentelemetry.io/collector/pdata/internal/data,Apache-2.0,Copyright The OpenTelemetry Authors @@ -194,15 +194,15 @@ pkg/otlp/metrics,go.opentelemetry.io/collector/pdata/pcommon,Apache-2.0,Copyrigh pkg/otlp/metrics,go.opentelemetry.io/collector/pdata/pmetric,Apache-2.0,Copyright The OpenTelemetry Authors pkg/otlp/metrics,go.opentelemetry.io/collector/pdata/pmetric/internal/pmetricjson,Apache-2.0,Copyright The OpenTelemetry Authors pkg/otlp/metrics,go.opentelemetry.io/collector/semconv/v1.6.1,Apache-2.0,Copyright The OpenTelemetry Authors -pkg/otlp/metrics,go.uber.org/atomic,MIT,"Copyright (c) 2016 Uber Technologies, Inc | Copyright (c) 2016 Uber Technologies, Inc" -pkg/otlp/metrics,go.uber.org/multierr,MIT,"Copyright (c) 2017-2021 Uber Technologies, Inc | Copyright (c) 2017-2021 Uber Technologies, Inc" -pkg/otlp/metrics,go.uber.org/zap,MIT,"Copyright (c) 2016-2017 Uber Technologies, Inc | Copyright (c) 2016-2017 Uber Technologies, Inc" -pkg/otlp/metrics,go.uber.org/zap/buffer,MIT,"Copyright (c) 2016-2017 Uber Technologies, Inc | Copyright (c) 2016-2017 Uber Technologies, Inc" -pkg/otlp/metrics,go.uber.org/zap/internal,MIT,"Copyright (c) 2016-2017 Uber Technologies, Inc | Copyright (c) 2016-2017 Uber Technologies, Inc" -pkg/otlp/metrics,go.uber.org/zap/internal/bufferpool,MIT,"Copyright (c) 2016-2017 Uber Technologies, Inc | Copyright (c) 2016-2017 Uber Technologies, Inc" -pkg/otlp/metrics,go.uber.org/zap/internal/color,MIT,"Copyright (c) 2016-2017 Uber Technologies, Inc | Copyright (c) 2016-2017 Uber Technologies, Inc" -pkg/otlp/metrics,go.uber.org/zap/internal/exit,MIT,"Copyright (c) 2016-2017 Uber Technologies, Inc | Copyright (c) 2016-2017 Uber Technologies, Inc" -pkg/otlp/metrics,go.uber.org/zap/zapcore,MIT,"Copyright (c) 2016-2017 Uber Technologies, Inc | Copyright (c) 2016-2017 Uber Technologies, Inc" +pkg/otlp/metrics,go.uber.org/atomic,MIT,"Copyright (c) 2016 Uber Technologies, Inc" +pkg/otlp/metrics,go.uber.org/multierr,MIT,"Copyright (c) 2017-2021 Uber Technologies, Inc" +pkg/otlp/metrics,go.uber.org/zap,MIT,"Copyright (c) 2016-2017 Uber Technologies, Inc" +pkg/otlp/metrics,go.uber.org/zap/buffer,MIT,"Copyright (c) 2016-2017 Uber Technologies, Inc" +pkg/otlp/metrics,go.uber.org/zap/internal,MIT,"Copyright (c) 2016-2017 Uber Technologies, Inc" +pkg/otlp/metrics,go.uber.org/zap/internal/bufferpool,MIT,"Copyright (c) 2016-2017 Uber Technologies, Inc" +pkg/otlp/metrics,go.uber.org/zap/internal/color,MIT,"Copyright (c) 2016-2017 Uber Technologies, Inc" +pkg/otlp/metrics,go.uber.org/zap/internal/exit,MIT,"Copyright (c) 2016-2017 Uber Technologies, Inc" +pkg/otlp/metrics,go.uber.org/zap/zapcore,MIT,"Copyright (c) 2016-2017 Uber Technologies, Inc" pkg/otlp/metrics,golang.org/x/exp/constraints,BSD-3-Clause,Copyright (c) 2009 The Go Authors. All rights reserved pkg/otlp/metrics,golang.org/x/exp/slices,BSD-3-Clause,Copyright (c) 2009 The Go Authors. All rights reserved pkg/otlp/metrics,golang.org/x/net/http/httpguts,BSD-3-Clause,Copyright (c) 2009 The Go Authors. All rights reserved From c1ad4a7eb54853f4ef1302ac1f7820a767adadd8 Mon Sep 17 00:00:00 2001 From: Stanley Liu Date: Wed, 5 Apr 2023 12:52:21 -0400 Subject: [PATCH 15/22] Fix lint --- pkg/otlp/metrics/metrics_translator_test.go | 34 +++++++-------------- 1 file changed, 11 insertions(+), 23 deletions(-) diff --git a/pkg/otlp/metrics/metrics_translator_test.go b/pkg/otlp/metrics/metrics_translator_test.go index 3faf2376..95f78a77 100644 --- a/pkg/otlp/metrics/metrics_translator_test.go +++ b/pkg/otlp/metrics/metrics_translator_test.go @@ -433,7 +433,7 @@ func TestMapSumRuntimeMetricWithAttributesHasMapping(t *testing.T) { key: "generation", values: []string{"gen0"}, }} - if err := tr.MapMetrics(ctx, createTestMetricWithAttributes(false, "process.runtime.dotnet.gc.collections.count", pmetric.MetricTypeSum, attributes), consumer); err != nil { + if err := tr.MapMetrics(ctx, createTestMetricWithAttributes("process.runtime.dotnet.gc.collections.count", pmetric.MetricTypeSum, attributes), consumer); err != nil { t.Fatal(err) } startTs := int(getProcessStartTime()) + 1 @@ -454,7 +454,7 @@ func TestMapGaugeRuntimeMetricWithAttributesHasMapping(t *testing.T) { key: "generation", values: []string{"gen1"}, }} - if err := tr.MapMetrics(ctx, createTestMetricWithAttributes(false, "process.runtime.dotnet.gc.heap.size", pmetric.MetricTypeGauge, attributes), consumer); err != nil { + if err := tr.MapMetrics(ctx, createTestMetricWithAttributes("process.runtime.dotnet.gc.heap.size", pmetric.MetricTypeGauge, attributes), consumer); err != nil { t.Fatal(err) } startTs := int(getProcessStartTime()) + 1 @@ -473,7 +473,7 @@ func TestMapHistogramRuntimeMetricHasMapping(t *testing.T) { consumer := &mockFullConsumer{} fmt.Println(consumer.metrics) - if err := tr.MapMetrics(ctx, createTestHistogramMetric(false, "process.runtime.jvm.gc.duration"), consumer); err != nil { + if err := tr.MapMetrics(ctx, createTestHistogramMetric("process.runtime.jvm.gc.duration"), consumer); err != nil { t.Fatal(err) } startTs := int(getProcessStartTime()) + 1 @@ -500,7 +500,7 @@ func TestMapHistogramRuntimeMetricWithAttributesHasMapping(t *testing.T) { key: "generation", values: []string{"gen1"}, }} - if err := tr.MapMetrics(ctx, createTestMetricWithAttributes(false, "process.runtime.dotnet.gc.heap.size", pmetric.MetricTypeHistogram, attributes), consumer); err != nil { + if err := tr.MapMetrics(ctx, createTestMetricWithAttributes("process.runtime.dotnet.gc.heap.size", pmetric.MetricTypeHistogram, attributes), consumer); err != nil { t.Fatal(err) } startTs := int(getProcessStartTime()) + 1 @@ -530,7 +530,7 @@ func TestMapRuntimeMetricWithTwoAttributesHasMapping(t *testing.T) { key: "type", values: []string{"heap"}, }} - if err := tr.MapMetrics(ctx, createTestMetricWithAttributes(false, "process.runtime.jvm.memory.usage", pmetric.MetricTypeGauge, attributes), consumer); err != nil { + if err := tr.MapMetrics(ctx, createTestMetricWithAttributes("process.runtime.jvm.memory.usage", pmetric.MetricTypeGauge, attributes), consumer); err != nil { t.Fatal(err) } startTs := int(getProcessStartTime()) + 1 @@ -552,7 +552,7 @@ func TestMapGaugeRuntimeMetricWithInvalidAttributes(t *testing.T) { key: "type", values: []string{"heap2"}, }} - if err := tr.MapMetrics(ctx, createTestMetricWithAttributes(false, "process.runtime.jvm.memory.usage", pmetric.MetricTypeGauge, attributes), consumer); err != nil { + if err := tr.MapMetrics(ctx, createTestMetricWithAttributes("process.runtime.jvm.memory.usage", pmetric.MetricTypeGauge, attributes), consumer); err != nil { t.Fatal(err) } startTs := int(getProcessStartTime()) + 1 @@ -927,7 +927,7 @@ func createTestDoubleCumulativeMonotonicMetrics(tsmatch bool) pmetric.Metrics { return md } -func createTestHistogramMetric(tsmatch bool, metricName string) pmetric.Metrics { +func createTestHistogramMetric(metricName string) pmetric.Metrics { md := pmetric.NewMetrics() met := md.ResourceMetrics().AppendEmpty().ScopeMetrics().AppendEmpty().Metrics().AppendEmpty() met.SetName(metricName) @@ -939,11 +939,7 @@ func createTestHistogramMetric(tsmatch bool, metricName string) pmetric.Metrics startTs := int(getProcessStartTime()) + 1 hpCount := hpsCount.AppendEmpty() hpCount.SetStartTimestamp(seconds(startTs)) - if tsmatch { - hpCount.SetTimestamp(seconds(startTs)) - } else { - hpCount.SetTimestamp(seconds(startTs + 1)) - } + hpCount.SetTimestamp(seconds(startTs + 1)) hpCount.ExplicitBounds().FromRaw([]float64{}) hpCount.BucketCounts().FromRaw([]uint64{100}) hpCount.SetCount(100) @@ -953,7 +949,7 @@ func createTestHistogramMetric(tsmatch bool, metricName string) pmetric.Metrics return md } -func createTestMetricWithAttributes(tsmatch bool, metricName string, metricType pmetric.MetricType, attributes []runtimeMetricAttribute) pmetric.Metrics { +func createTestMetricWithAttributes(metricName string, metricType pmetric.MetricType, attributes []runtimeMetricAttribute) pmetric.Metrics { md := pmetric.NewMetrics() met := md.ResourceMetrics().AppendEmpty().ScopeMetrics().AppendEmpty().Metrics().AppendEmpty() met.SetName(metricName) @@ -981,11 +977,7 @@ func createTestMetricWithAttributes(tsmatch bool, metricName string, metricType dpInt.Attributes().PutStr(attr.key, attr.values[0]) } dpInt.SetStartTimestamp(seconds(startTs)) - if tsmatch { - dpInt.SetTimestamp(seconds(startTs)) - } else { - dpInt.SetTimestamp(seconds(startTs + 1)) - } + dpInt.SetTimestamp(seconds(startTs + 1)) dpInt.SetIntValue(10) return md } @@ -997,11 +989,7 @@ func createTestMetricWithAttributes(tsmatch bool, metricName string, metricType hpCount.Attributes().PutStr(attr.key, attr.values[0]) } hpCount.SetStartTimestamp(seconds(startTs)) - if tsmatch { - hpCount.SetTimestamp(seconds(startTs)) - } else { - hpCount.SetTimestamp(seconds(startTs + 1)) - } + hpCount.SetTimestamp(seconds(startTs + 1)) hpCount.ExplicitBounds().FromRaw([]float64{}) hpCount.BucketCounts().FromRaw([]uint64{100}) hpCount.SetCount(100) From ff402cc487da1ffd70f5a6293c78e6b83ff6d293 Mon Sep 17 00:00:00 2001 From: Stanley Liu Date: Mon, 10 Apr 2023 14:01:52 -0400 Subject: [PATCH 16/22] Add type for runtimeMetricMappingList --- pkg/otlp/metrics/runtime_metric_mappings.go | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/pkg/otlp/metrics/runtime_metric_mappings.go b/pkg/otlp/metrics/runtime_metric_mappings.go index 15b8d92b..048de123 100644 --- a/pkg/otlp/metrics/runtime_metric_mappings.go +++ b/pkg/otlp/metrics/runtime_metric_mappings.go @@ -15,7 +15,11 @@ type runtimeMetricAttribute struct { values []string // the attribute value, or multiple values if there is more than one value for the same mapping } -var goRuntimeMetricsMappings = map[string][]runtimeMetricMapping{ +// runtimeMetricMappingList defines the structure for a list of runtime metric mappings where the key +// represents the OTel metric name and the runtimeMetricMapping contains the Datadog metric name +type runtimeMetricMappingList map[string][]runtimeMetricMapping + +var goRuntimeMetricsMappings = runtimeMetricMappingList{ "process.runtime.go.goroutines": {{mappedName: "runtime.go.num_goroutine"}}, "process.runtime.go.cgo.calls": {{mappedName: "runtime.go.num_cgo_call"}}, "process.runtime.go.lookups": {{mappedName: "runtime.go.mem_stats.lookups"}}, @@ -29,7 +33,7 @@ var goRuntimeMetricsMappings = map[string][]runtimeMetricMapping{ "process.runtime.go.gc.count": {{mappedName: "runtime.go.mem_stats.num_gc"}}, } -var dotnetRuntimeMetricsMappings = map[string][]runtimeMetricMapping{ +var dotnetRuntimeMetricsMappings = runtimeMetricMappingList{ "process.runtime.dotnet.thread_pool.threads.count": {{mappedName: "runtime.dotnet.threads.count"}}, "process.runtime.dotnet.monitor.lock_contention.count": {{mappedName: "runtime.dotnet.threads.contention_count"}}, "process.runtime.dotnet.exceptions.count": {{mappedName: "runtime.dotnet.exceptions.count"}}, @@ -79,7 +83,7 @@ var dotnetRuntimeMetricsMappings = map[string][]runtimeMetricMapping{ }}, } -var javaRuntimeMetricsMappings = map[string][]runtimeMetricMapping{ +var javaRuntimeMetricsMappings = runtimeMetricMappingList{ "process.runtime.jvm.threads.count": {{mappedName: "jvm.thread_count"}}, "process.runtime.jvm.classes.loaded": {{mappedName: "jvm.loaded_classes"}}, "process.runtime.jvm.system.cpu.utilization": {{mappedName: "jvm.cpu_load.system"}}, @@ -214,8 +218,8 @@ var javaRuntimeMetricsMappings = map[string][]runtimeMetricMapping{ }}, } -func getRuntimeMetricsMappings() map[string][]runtimeMetricMapping { - res := map[string][]runtimeMetricMapping{} +func getRuntimeMetricsMappings() runtimeMetricMappingList { + res := runtimeMetricMappingList{} for k, v := range goRuntimeMetricsMappings { res[k] = v } From 189b7c8efa9725bf275d6db79ebc47fc7268c635 Mon Sep 17 00:00:00 2001 From: Stanley Liu Date: Mon, 10 Apr 2023 14:55:19 -0400 Subject: [PATCH 17/22] Debug logs --- pkg/otlp/metrics/metrics_translator.go | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/pkg/otlp/metrics/metrics_translator.go b/pkg/otlp/metrics/metrics_translator.go index b1415a6e..eb1f814e 100644 --- a/pkg/otlp/metrics/metrics_translator.go +++ b/pkg/otlp/metrics/metrics_translator.go @@ -456,8 +456,16 @@ func mapGaugeRuntimeMetricWithAttributes(md pmetric.Metric, metricsArray pmetric matchesAttributes := true for _, attribute := range mp.attributes { attributeValue, res := md.Gauge().DataPoints().At(i).Attributes().Get(attribute.key) + if md.Name() == "process.runtime.jvm.memory.usage" { + fmt.Println("-----------------------") + fmt.Printf("attributeValue: %v\n", attributeValue.AsString()) + fmt.Printf("attribute.key: %v\n", attribute.key) + fmt.Printf("attribute.values: %v\n", attribute.values) + fmt.Println("-----------------------") + } if !res || !slices.Contains(attribute.values, attributeValue.AsString()) { matchesAttributes = false + fmt.Println("matchesAttributes false") break } } From 1c6fbaeea05c7c3b493ad1f656ec29ef48537016 Mon Sep 17 00:00:00 2001 From: Stanley Liu Date: Mon, 10 Apr 2023 15:05:54 -0400 Subject: [PATCH 18/22] Debug logs --- pkg/otlp/metrics/metrics_translator.go | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/pkg/otlp/metrics/metrics_translator.go b/pkg/otlp/metrics/metrics_translator.go index eb1f814e..a179e6cb 100644 --- a/pkg/otlp/metrics/metrics_translator.go +++ b/pkg/otlp/metrics/metrics_translator.go @@ -456,13 +456,12 @@ func mapGaugeRuntimeMetricWithAttributes(md pmetric.Metric, metricsArray pmetric matchesAttributes := true for _, attribute := range mp.attributes { attributeValue, res := md.Gauge().DataPoints().At(i).Attributes().Get(attribute.key) - if md.Name() == "process.runtime.jvm.memory.usage" { - fmt.Println("-----------------------") - fmt.Printf("attributeValue: %v\n", attributeValue.AsString()) - fmt.Printf("attribute.key: %v\n", attribute.key) - fmt.Printf("attribute.values: %v\n", attribute.values) - fmt.Println("-----------------------") - } + fmt.Println("-----------------------") + fmt.Printf("md.Name: %v\n", md.Name()) + fmt.Printf("attributeValue: %v\n", attributeValue.AsString()) + fmt.Printf("attribute.key: %v\n", attribute.key) + fmt.Printf("attribute.values: %v\n", attribute.values) + fmt.Println("-----------------------") if !res || !slices.Contains(attribute.values, attributeValue.AsString()) { matchesAttributes = false fmt.Println("matchesAttributes false") @@ -494,7 +493,14 @@ func mapSumRuntimeMetricWithAttributes(md pmetric.Metric, metricsArray pmetric.M matchesAttributes := true for _, attribute := range mp.attributes { attributeValue, res := md.Sum().DataPoints().At(i).Attributes().Get(attribute.key) + fmt.Println("-----------------------") + fmt.Printf("md.Name: %v\n", md.Name()) + fmt.Printf("attributeValue: %v\n", attributeValue.AsString()) + fmt.Printf("attribute.key: %v\n", attribute.key) + fmt.Printf("attribute.values: %v\n", attribute.values) + fmt.Println("-----------------------") if !res || !slices.Contains(attribute.values, attributeValue.AsString()) { + fmt.Println("matchesAttributes false") matchesAttributes = false break } From 907b92c0207b4f3eb5301a5cfa7f66cc4878af90 Mon Sep 17 00:00:00 2001 From: Stanley Liu Date: Mon, 10 Apr 2023 15:23:21 -0400 Subject: [PATCH 19/22] Use Str instead of AsString --- pkg/otlp/metrics/metrics_translator.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/pkg/otlp/metrics/metrics_translator.go b/pkg/otlp/metrics/metrics_translator.go index a179e6cb..0a5aab35 100644 --- a/pkg/otlp/metrics/metrics_translator.go +++ b/pkg/otlp/metrics/metrics_translator.go @@ -458,11 +458,11 @@ func mapGaugeRuntimeMetricWithAttributes(md pmetric.Metric, metricsArray pmetric attributeValue, res := md.Gauge().DataPoints().At(i).Attributes().Get(attribute.key) fmt.Println("-----------------------") fmt.Printf("md.Name: %v\n", md.Name()) - fmt.Printf("attributeValue: %v\n", attributeValue.AsString()) + fmt.Printf("attributeValue: %v\n", attributeValue.Str()) fmt.Printf("attribute.key: %v\n", attribute.key) fmt.Printf("attribute.values: %v\n", attribute.values) fmt.Println("-----------------------") - if !res || !slices.Contains(attribute.values, attributeValue.AsString()) { + if !res || !slices.Contains(attribute.values, attributeValue.Str()) { matchesAttributes = false fmt.Println("matchesAttributes false") break @@ -495,11 +495,11 @@ func mapSumRuntimeMetricWithAttributes(md pmetric.Metric, metricsArray pmetric.M attributeValue, res := md.Sum().DataPoints().At(i).Attributes().Get(attribute.key) fmt.Println("-----------------------") fmt.Printf("md.Name: %v\n", md.Name()) - fmt.Printf("attributeValue: %v\n", attributeValue.AsString()) + fmt.Printf("attributeValue: %v\n", attributeValue.Str()) fmt.Printf("attribute.key: %v\n", attribute.key) fmt.Printf("attribute.values: %v\n", attribute.values) fmt.Println("-----------------------") - if !res || !slices.Contains(attribute.values, attributeValue.AsString()) { + if !res || !slices.Contains(attribute.values, attributeValue.Str()) { fmt.Println("matchesAttributes false") matchesAttributes = false break @@ -532,7 +532,7 @@ func mapHistogramRuntimeMetricWithAttributes(md pmetric.Metric, metricsArray pme matchesAttributes := true for _, attribute := range mp.attributes { attributeValue, res := md.Histogram().DataPoints().At(i).Attributes().Get(attribute.key) - if !res || !slices.Contains(attribute.values, attributeValue.AsString()) { + if !res || !slices.Contains(attribute.values, attributeValue.Str()) { matchesAttributes = false break } From 6c52b22260602b260c59aba03a2125c8c7b7acbe Mon Sep 17 00:00:00 2001 From: Stanley Liu Date: Mon, 10 Apr 2023 15:49:00 -0400 Subject: [PATCH 20/22] Change attribute values --- pkg/otlp/metrics/metrics_translator.go | 20 ++++++++++---------- pkg/otlp/metrics/runtime_metric_mappings.go | 8 ++++---- 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/pkg/otlp/metrics/metrics_translator.go b/pkg/otlp/metrics/metrics_translator.go index 0a5aab35..df3289f8 100644 --- a/pkg/otlp/metrics/metrics_translator.go +++ b/pkg/otlp/metrics/metrics_translator.go @@ -458,11 +458,11 @@ func mapGaugeRuntimeMetricWithAttributes(md pmetric.Metric, metricsArray pmetric attributeValue, res := md.Gauge().DataPoints().At(i).Attributes().Get(attribute.key) fmt.Println("-----------------------") fmt.Printf("md.Name: %v\n", md.Name()) - fmt.Printf("attributeValue: %v\n", attributeValue.Str()) + fmt.Printf("attributeValue: %v\n", attributeValue.AsString()) fmt.Printf("attribute.key: %v\n", attribute.key) fmt.Printf("attribute.values: %v\n", attribute.values) fmt.Println("-----------------------") - if !res || !slices.Contains(attribute.values, attributeValue.Str()) { + if !res || !slices.Contains(attribute.values, attributeValue.AsString()) { matchesAttributes = false fmt.Println("matchesAttributes false") break @@ -493,13 +493,13 @@ func mapSumRuntimeMetricWithAttributes(md pmetric.Metric, metricsArray pmetric.M matchesAttributes := true for _, attribute := range mp.attributes { attributeValue, res := md.Sum().DataPoints().At(i).Attributes().Get(attribute.key) - fmt.Println("-----------------------") - fmt.Printf("md.Name: %v\n", md.Name()) - fmt.Printf("attributeValue: %v\n", attributeValue.Str()) - fmt.Printf("attribute.key: %v\n", attribute.key) - fmt.Printf("attribute.values: %v\n", attribute.values) - fmt.Println("-----------------------") - if !res || !slices.Contains(attribute.values, attributeValue.Str()) { + //fmt.Println("-----------------------") + //fmt.Printf("md.Name: %v\n", md.Name()) + //fmt.Printf("attributeValue: %v\n", attributeValue.AsString()) + //fmt.Printf("attribute.key: %v\n", attribute.key) + //fmt.Printf("attribute.values: %v\n", attribute.values) + //fmt.Println("-----------------------") + if !res || !slices.Contains(attribute.values, attributeValue.AsString()) { fmt.Println("matchesAttributes false") matchesAttributes = false break @@ -532,7 +532,7 @@ func mapHistogramRuntimeMetricWithAttributes(md pmetric.Metric, metricsArray pme matchesAttributes := true for _, attribute := range mp.attributes { attributeValue, res := md.Histogram().DataPoints().At(i).Attributes().Get(attribute.key) - if !res || !slices.Contains(attribute.values, attributeValue.Str()) { + if !res || !slices.Contains(attribute.values, attributeValue.AsString()) { matchesAttributes = false break } diff --git a/pkg/otlp/metrics/runtime_metric_mappings.go b/pkg/otlp/metrics/runtime_metric_mappings.go index 048de123..b751a391 100644 --- a/pkg/otlp/metrics/runtime_metric_mappings.go +++ b/pkg/otlp/metrics/runtime_metric_mappings.go @@ -105,7 +105,7 @@ var javaRuntimeMetricsMappings = runtimeMetricMappingList{ mappedName: "jvm.gc.old_gen_size", attributes: []runtimeMetricAttribute{{ key: "pool", - values: []string{"g1_old_gen", "tenured_gen", "ps_old_gen"}, + values: []string{"G1 Old Gen", "Tenured Gen", "PS Old Gen"}, }, { key: "type", values: []string{"heap"}, @@ -114,7 +114,7 @@ var javaRuntimeMetricsMappings = runtimeMetricMappingList{ mappedName: "jvm.gc.eden_size", attributes: []runtimeMetricAttribute{{ key: "pool", - values: []string{"g1_eden_space", "eden_space", "par_eden_space", "ps_eden_space"}, + values: []string{"G1 Eden Space", "Eden Space", "Par Eden Space", "PS Eden Space"}, }, { key: "type", values: []string{"heap"}, @@ -123,7 +123,7 @@ var javaRuntimeMetricsMappings = runtimeMetricMappingList{ mappedName: "jvm.gc.survivor_size", attributes: []runtimeMetricAttribute{{ key: "pool", - values: []string{"g1_survivor_space", "survivor_space", "par_survivor_space", "ps_survivor_space"}, + values: []string{"G1 Survivor Space", "Survivor Space", "Par Survivor Space", "PS Survivor Space"}, }, { key: "type", values: []string{"heap"}, @@ -132,7 +132,7 @@ var javaRuntimeMetricsMappings = runtimeMetricMappingList{ mappedName: "jvm.gc.metaspace_size", attributes: []runtimeMetricAttribute{{ key: "pool", - values: []string{"metaspace"}, + values: []string{"Metaspace"}, }, { key: "type", values: []string{"non_heap"}, From aad19eabc8991a3143594e2b0f75b9649b3f9c00 Mon Sep 17 00:00:00 2001 From: Stanley Liu Date: Mon, 10 Apr 2023 16:00:52 -0400 Subject: [PATCH 21/22] Remove debug logs --- pkg/otlp/metrics/metrics_translator.go | 14 -------------- pkg/otlp/metrics/metrics_translator_test.go | 6 +++--- 2 files changed, 3 insertions(+), 17 deletions(-) diff --git a/pkg/otlp/metrics/metrics_translator.go b/pkg/otlp/metrics/metrics_translator.go index df3289f8..b1415a6e 100644 --- a/pkg/otlp/metrics/metrics_translator.go +++ b/pkg/otlp/metrics/metrics_translator.go @@ -456,15 +456,8 @@ func mapGaugeRuntimeMetricWithAttributes(md pmetric.Metric, metricsArray pmetric matchesAttributes := true for _, attribute := range mp.attributes { attributeValue, res := md.Gauge().DataPoints().At(i).Attributes().Get(attribute.key) - fmt.Println("-----------------------") - fmt.Printf("md.Name: %v\n", md.Name()) - fmt.Printf("attributeValue: %v\n", attributeValue.AsString()) - fmt.Printf("attribute.key: %v\n", attribute.key) - fmt.Printf("attribute.values: %v\n", attribute.values) - fmt.Println("-----------------------") if !res || !slices.Contains(attribute.values, attributeValue.AsString()) { matchesAttributes = false - fmt.Println("matchesAttributes false") break } } @@ -493,14 +486,7 @@ func mapSumRuntimeMetricWithAttributes(md pmetric.Metric, metricsArray pmetric.M matchesAttributes := true for _, attribute := range mp.attributes { attributeValue, res := md.Sum().DataPoints().At(i).Attributes().Get(attribute.key) - //fmt.Println("-----------------------") - //fmt.Printf("md.Name: %v\n", md.Name()) - //fmt.Printf("attributeValue: %v\n", attributeValue.AsString()) - //fmt.Printf("attribute.key: %v\n", attribute.key) - //fmt.Printf("attribute.values: %v\n", attribute.values) - //fmt.Println("-----------------------") if !res || !slices.Contains(attribute.values, attributeValue.AsString()) { - fmt.Println("matchesAttributes false") matchesAttributes = false break } diff --git a/pkg/otlp/metrics/metrics_translator_test.go b/pkg/otlp/metrics/metrics_translator_test.go index 95f78a77..23896d0f 100644 --- a/pkg/otlp/metrics/metrics_translator_test.go +++ b/pkg/otlp/metrics/metrics_translator_test.go @@ -525,7 +525,7 @@ func TestMapRuntimeMetricWithTwoAttributesHasMapping(t *testing.T) { consumer := &mockFullConsumer{} attributes := []runtimeMetricAttribute{{ key: "pool", - values: []string{"g1_old_gen"}, + values: []string{"G1 Old Gen"}, }, { key: "type", values: []string{"heap"}, @@ -537,8 +537,8 @@ func TestMapRuntimeMetricWithTwoAttributesHasMapping(t *testing.T) { assert.ElementsMatch(t, consumer.metrics, []metric{ - newGaugeWithHost(newDims("process.runtime.jvm.memory.usage").AddTags("pool:g1_old_gen", "type:heap"), uint64(seconds(startTs+1)), 10, fallbackHostname), - newGaugeWithHost(newDims("jvm.heap_memory").AddTags("pool:g1_old_gen"), uint64(seconds(startTs+1)), 10, fallbackHostname), + newGaugeWithHost(newDims("process.runtime.jvm.memory.usage").AddTags("pool:G1 Old Gen", "type:heap"), uint64(seconds(startTs+1)), 10, fallbackHostname), + newGaugeWithHost(newDims("jvm.heap_memory").AddTags("pool:G1 Old Gen"), uint64(seconds(startTs+1)), 10, fallbackHostname), newGaugeWithHost(newDims("jvm.gc.old_gen_size"), uint64(seconds(startTs+1)), 10, fallbackHostname), }, ) From a777b85d8f82f5a806e50eeee7a70c9d02e0b667 Mon Sep 17 00:00:00 2001 From: Stanley Liu Date: Tue, 11 Apr 2023 15:50:40 -0400 Subject: [PATCH 22/22] Remove break in matchesAttributes section to map all DataPoints --- pkg/otlp/metrics/metrics_translator.go | 2 - pkg/otlp/metrics/metrics_translator_test.go | 89 ++++++++++++++------- 2 files changed, 62 insertions(+), 29 deletions(-) diff --git a/pkg/otlp/metrics/metrics_translator.go b/pkg/otlp/metrics/metrics_translator.go index b1415a6e..0f4525a7 100644 --- a/pkg/otlp/metrics/metrics_translator.go +++ b/pkg/otlp/metrics/metrics_translator.go @@ -475,7 +475,6 @@ func mapGaugeRuntimeMetricWithAttributes(md pmetric.Metric, metricsArray pmetric return false }) cp.SetName(mp.mappedName) - break } } } @@ -507,7 +506,6 @@ func mapSumRuntimeMetricWithAttributes(md pmetric.Metric, metricsArray pmetric.M return false }) cp.SetName(mp.mappedName) - break } } } diff --git a/pkg/otlp/metrics/metrics_translator_test.go b/pkg/otlp/metrics/metrics_translator_test.go index 23896d0f..83829950 100644 --- a/pkg/otlp/metrics/metrics_translator_test.go +++ b/pkg/otlp/metrics/metrics_translator_test.go @@ -433,7 +433,7 @@ func TestMapSumRuntimeMetricWithAttributesHasMapping(t *testing.T) { key: "generation", values: []string{"gen0"}, }} - if err := tr.MapMetrics(ctx, createTestMetricWithAttributes("process.runtime.dotnet.gc.collections.count", pmetric.MetricTypeSum, attributes), consumer); err != nil { + if err := tr.MapMetrics(ctx, createTestMetricWithAttributes("process.runtime.dotnet.gc.collections.count", pmetric.MetricTypeSum, attributes, 1), consumer); err != nil { t.Fatal(err) } startTs := int(getProcessStartTime()) + 1 @@ -454,7 +454,7 @@ func TestMapGaugeRuntimeMetricWithAttributesHasMapping(t *testing.T) { key: "generation", values: []string{"gen1"}, }} - if err := tr.MapMetrics(ctx, createTestMetricWithAttributes("process.runtime.dotnet.gc.heap.size", pmetric.MetricTypeGauge, attributes), consumer); err != nil { + if err := tr.MapMetrics(ctx, createTestMetricWithAttributes("process.runtime.dotnet.gc.heap.size", pmetric.MetricTypeGauge, attributes, 1), consumer); err != nil { t.Fatal(err) } startTs := int(getProcessStartTime()) + 1 @@ -500,7 +500,7 @@ func TestMapHistogramRuntimeMetricWithAttributesHasMapping(t *testing.T) { key: "generation", values: []string{"gen1"}, }} - if err := tr.MapMetrics(ctx, createTestMetricWithAttributes("process.runtime.dotnet.gc.heap.size", pmetric.MetricTypeHistogram, attributes), consumer); err != nil { + if err := tr.MapMetrics(ctx, createTestMetricWithAttributes("process.runtime.dotnet.gc.heap.size", pmetric.MetricTypeHistogram, attributes, 1), consumer); err != nil { t.Fatal(err) } startTs := int(getProcessStartTime()) + 1 @@ -530,7 +530,7 @@ func TestMapRuntimeMetricWithTwoAttributesHasMapping(t *testing.T) { key: "type", values: []string{"heap"}, }} - if err := tr.MapMetrics(ctx, createTestMetricWithAttributes("process.runtime.jvm.memory.usage", pmetric.MetricTypeGauge, attributes), consumer); err != nil { + if err := tr.MapMetrics(ctx, createTestMetricWithAttributes("process.runtime.jvm.memory.usage", pmetric.MetricTypeGauge, attributes, 1), consumer); err != nil { t.Fatal(err) } startTs := int(getProcessStartTime()) + 1 @@ -544,6 +544,37 @@ func TestMapRuntimeMetricWithTwoAttributesHasMapping(t *testing.T) { ) } +func TestMapRuntimeMetricWithTwoAttributesMultipleDataPointsHasMapping(t *testing.T) { + ctx := context.Background() + tr := newTranslator(t, zap.NewNop()) + consumer := &mockFullConsumer{} + attributes := []runtimeMetricAttribute{{ + key: "pool", + values: []string{"G1 Old Gen", "G1 Survivor Space", "G1 Eden Space"}, + }, { + key: "type", + values: []string{"heap", "heap", "heap"}, + }} + if err := tr.MapMetrics(ctx, createTestMetricWithAttributes("process.runtime.jvm.memory.usage", pmetric.MetricTypeGauge, attributes, 3), consumer); err != nil { + t.Fatal(err) + } + startTs := int(getProcessStartTime()) + 1 + assert.ElementsMatch(t, + consumer.metrics, + []metric{ + newGaugeWithHost(newDims("process.runtime.jvm.memory.usage").AddTags("pool:G1 Old Gen", "type:heap"), uint64(seconds(startTs+1)), 10, fallbackHostname), + newGaugeWithHost(newDims("process.runtime.jvm.memory.usage").AddTags("pool:G1 Survivor Space", "type:heap"), uint64(seconds(startTs+2)), 20, fallbackHostname), + newGaugeWithHost(newDims("process.runtime.jvm.memory.usage").AddTags("pool:G1 Eden Space", "type:heap"), uint64(seconds(startTs+3)), 30, fallbackHostname), + newGaugeWithHost(newDims("jvm.heap_memory").AddTags("pool:G1 Old Gen"), uint64(seconds(startTs+1)), 10, fallbackHostname), + newGaugeWithHost(newDims("jvm.heap_memory").AddTags("pool:G1 Survivor Space"), uint64(seconds(startTs+2)), 20, fallbackHostname), + newGaugeWithHost(newDims("jvm.heap_memory").AddTags("pool:G1 Eden Space"), uint64(seconds(startTs+3)), 30, fallbackHostname), + newGaugeWithHost(newDims("jvm.gc.old_gen_size"), uint64(seconds(startTs+1)), 10, fallbackHostname), + newGaugeWithHost(newDims("jvm.gc.survivor_size"), uint64(seconds(startTs+2)), 20, fallbackHostname), + newGaugeWithHost(newDims("jvm.gc.eden_size"), uint64(seconds(startTs+3)), 30, fallbackHostname), + }, + ) +} + func TestMapGaugeRuntimeMetricWithInvalidAttributes(t *testing.T) { ctx := context.Background() tr := newTranslator(t, zap.NewNop()) @@ -552,7 +583,7 @@ func TestMapGaugeRuntimeMetricWithInvalidAttributes(t *testing.T) { key: "type", values: []string{"heap2"}, }} - if err := tr.MapMetrics(ctx, createTestMetricWithAttributes("process.runtime.jvm.memory.usage", pmetric.MetricTypeGauge, attributes), consumer); err != nil { + if err := tr.MapMetrics(ctx, createTestMetricWithAttributes("process.runtime.jvm.memory.usage", pmetric.MetricTypeGauge, attributes, 1), consumer); err != nil { t.Fatal(err) } startTs := int(getProcessStartTime()) + 1 @@ -949,7 +980,7 @@ func createTestHistogramMetric(metricName string) pmetric.Metrics { return md } -func createTestMetricWithAttributes(metricName string, metricType pmetric.MetricType, attributes []runtimeMetricAttribute) pmetric.Metrics { +func createTestMetricWithAttributes(metricName string, metricType pmetric.MetricType, attributes []runtimeMetricAttribute, dataPoints int) pmetric.Metrics { md := pmetric.NewMetrics() met := md.ResourceMetrics().AppendEmpty().ScopeMetrics().AppendEmpty().Metrics().AppendEmpty() met.SetName(metricName) @@ -970,32 +1001,36 @@ func createTestMetricWithAttributes(metricName string, metricType pmetric.Metric } if metricType != pmetric.MetricTypeHistogram { - dpsInt.EnsureCapacity(1) - startTs := int(getProcessStartTime()) + 1 - dpInt := dpsInt.AppendEmpty() - for _, attr := range attributes { - dpInt.Attributes().PutStr(attr.key, attr.values[0]) + dpsInt.EnsureCapacity(dataPoints) + for i := 0; i < dataPoints; i++ { + startTs := int(getProcessStartTime()) + 1 + dpInt := dpsInt.AppendEmpty() + for _, attr := range attributes { + dpInt.Attributes().PutStr(attr.key, attr.values[i]) + } + dpInt.SetStartTimestamp(seconds(startTs)) + dpInt.SetTimestamp(seconds(startTs + 1 + i)) + dpInt.SetIntValue(int64(10 * (1 + i))) } - dpInt.SetStartTimestamp(seconds(startTs)) - dpInt.SetTimestamp(seconds(startTs + 1)) - dpInt.SetIntValue(10) return md } - hpsCount.EnsureCapacity(1) - startTs := int(getProcessStartTime()) + 1 - hpCount := hpsCount.AppendEmpty() - for _, attr := range attributes { - hpCount.Attributes().PutStr(attr.key, attr.values[0]) + hpsCount.EnsureCapacity(dataPoints) + for i := 0; i < dataPoints; i++ { + startTs := int(getProcessStartTime()) + 1 + hpCount := hpsCount.AppendEmpty() + for _, attr := range attributes { + hpCount.Attributes().PutStr(attr.key, attr.values[i]) + } + hpCount.SetStartTimestamp(seconds(startTs)) + hpCount.SetTimestamp(seconds(startTs + 1 + i)) + hpCount.ExplicitBounds().FromRaw([]float64{}) + hpCount.BucketCounts().FromRaw([]uint64{100}) + hpCount.SetCount(100) + hpCount.SetSum(0) + hpCount.SetMin(-100) + hpCount.SetMax(100) } - hpCount.SetStartTimestamp(seconds(startTs)) - hpCount.SetTimestamp(seconds(startTs + 1)) - hpCount.ExplicitBounds().FromRaw([]float64{}) - hpCount.BucketCounts().FromRaw([]uint64{100}) - hpCount.SetCount(100) - hpCount.SetSum(0) - hpCount.SetMin(-100) - hpCount.SetMax(100) return md }