Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add support for OTel Java runtime metrics #19

Merged
merged 24 commits into from
Apr 12, 2023
Merged

Add support for OTel Java runtime metrics #19

Show file tree
Hide file tree
Changes from 7 commits
Commits
Show all changes
24 commits
Select commit Hold shift + click to select a range
1311115
Add java mappings
liustanley Mar 1, 2023
ac8084e
Debug logs
liustanley Mar 1, 2023
30a7441
More Debug logs
liustanley Mar 1, 2023
fbc42c3
Change to use md type
liustanley Mar 1, 2023
714ac65
Change to non_heap and remove metricType
liustanley Mar 1, 2023
367294d
Add unit test
liustanley Mar 3, 2023
d87dbf8
Refactor runtime metric mappings
liustanley Mar 6, 2023
f81d2a8
Merge branch 'main' into stanley.liu/otel-java-runtime-metrics
liustanley Mar 30, 2023
9e66fa7
Change runtime metric attribute definition
liustanley Apr 4, 2023
9ab5c81
Handle metrics with multiple attributes, add code to remove mapping a…
liustanley Apr 4, 2023
553fc22
Move runtime metric mappings to separate file
liustanley Apr 4, 2023
71814bc
Generate licenses
liustanley Apr 4, 2023
58a4e4d
Support mapping histograms, add rest of jvm mappings
liustanley Apr 4, 2023
6109fbd
Merge branch 'main' into stanley.liu/otel-java-runtime-metrics
liustanley Apr 4, 2023
5127316
Generate licenses
liustanley Apr 4, 2023
5e3cce8
License patch
liustanley Apr 5, 2023
c1ad4a7
Fix lint
liustanley Apr 5, 2023
ff402cc
Add type for runtimeMetricMappingList
liustanley Apr 10, 2023
189b7c8
Debug logs
liustanley Apr 10, 2023
1c6fbae
Debug logs
liustanley Apr 10, 2023
907b92c
Use Str instead of AsString
liustanley Apr 10, 2023
6c52b22
Change attribute values
liustanley Apr 10, 2023
aad19ea
Remove debug logs
liustanley Apr 10, 2023
a777b85
Remove break in matchesAttributes section to map all DataPoints
liustanley Apr 11, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
100 changes: 76 additions & 24 deletions pkg/otlp/metrics/metrics_translator.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -39,65 +39,117 @@ type runtimeMetricMapping struct {
mappedName string // the Datadog runtime metric name
attribute string // the name of the attribute this metric originates from
attributeValue string // the value of the above attribute that corresponds with this metric
metricType pmetric.MetricType
}

// runtimeMetricsMappings defines the mappings from OTel runtime metric names to their
// equivalent Datadog runtime metric names
var runtimeMetricsMappings = map[string][]runtimeMetricMapping{
"process.runtime.go.goroutines": {{mappedName: "runtime.go.num_goroutine"}},
"process.runtime.go.cgo.calls": {{mappedName: "runtime.go.num_cgo_call"}},
"process.runtime.go.lookups": {{mappedName: "runtime.go.mem_stats.lookups"}},
"process.runtime.go.mem.heap_alloc": {{mappedName: "runtime.go.mem_stats.heap_alloc"}},
"process.runtime.go.mem.heap_sys": {{mappedName: "runtime.go.mem_stats.heap_sys"}},
"process.runtime.go.mem.heap_idle": {{mappedName: "runtime.go.mem_stats.heap_idle"}},
"process.runtime.go.mem.heap_inuse": {{mappedName: "runtime.go.mem_stats.heap_inuse"}},
"process.runtime.go.mem.heap_released": {{mappedName: "runtime.go.mem_stats.heap_released"}},
"process.runtime.go.mem.heap_objects": {{mappedName: "runtime.go.mem_stats.heap_objects"}},
"process.runtime.go.gc.pause_total_ns": {{mappedName: "runtime.go.mem_stats.pause_total_ns"}},
"process.runtime.go.gc.count": {{mappedName: "runtime.go.mem_stats.num_gc"}},
var goRuntimeMetricsMappings = map[string][]runtimeMetricMapping{
"process.runtime.go.goroutines": {{mappedName: "runtime.go.num_goroutine"}},
"process.runtime.go.cgo.calls": {{mappedName: "runtime.go.num_cgo_call"}},
"process.runtime.go.lookups": {{mappedName: "runtime.go.mem_stats.lookups"}},
"process.runtime.go.mem.heap_alloc": {{mappedName: "runtime.go.mem_stats.heap_alloc"}},
"process.runtime.go.mem.heap_sys": {{mappedName: "runtime.go.mem_stats.heap_sys"}},
"process.runtime.go.mem.heap_idle": {{mappedName: "runtime.go.mem_stats.heap_idle"}},
"process.runtime.go.mem.heap_inuse": {{mappedName: "runtime.go.mem_stats.heap_inuse"}},
"process.runtime.go.mem.heap_released": {{mappedName: "runtime.go.mem_stats.heap_released"}},
"process.runtime.go.mem.heap_objects": {{mappedName: "runtime.go.mem_stats.heap_objects"}},
"process.runtime.go.gc.pause_total_ns": {{mappedName: "runtime.go.mem_stats.pause_total_ns"}},
"process.runtime.go.gc.count": {{mappedName: "runtime.go.mem_stats.num_gc"}},
}

var dotnetRuntimeMetricsMappings = map[string][]runtimeMetricMapping{
"process.runtime.dotnet.thread_pool.threads.count": {{mappedName: "runtime.dotnet.threads.count"}},
"process.runtime.dotnet.monitor.lock_contention.count": {{mappedName: "runtime.dotnet.threads.contention_count"}},
"process.runtime.dotnet.exceptions.count": {{mappedName: "runtime.dotnet.exceptions.count"}},
"process.runtime.dotnet.gc.heap.size": {{
mappedName: "runtime.dotnet.gc.size.gen0",
attribute: "generation",
attributeValue: "gen0",
metricType: pmetric.MetricTypeGauge,
}, {
mappedName: "runtime.dotnet.gc.size.gen1",
attribute: "generation",
attributeValue: "gen1",
metricType: pmetric.MetricTypeGauge,
}, {
mappedName: "runtime.dotnet.gc.size.gen2",
attribute: "generation",
attributeValue: "gen2",
metricType: pmetric.MetricTypeGauge,
}, {
mappedName: "runtime.dotnet.gc.size.loh",
attribute: "generation",
attributeValue: "loh",
metricType: pmetric.MetricTypeGauge,
}},
"process.runtime.dotnet.gc.collections.count": {{
mappedName: "runtime.dotnet.gc.count.gen0",
attribute: "generation",
attributeValue: "gen0",
metricType: pmetric.MetricTypeSum,
}, {
mappedName: "runtime.dotnet.gc.count.gen1",
attribute: "generation",
attributeValue: "gen1",
metricType: pmetric.MetricTypeSum,
}, {
mappedName: "runtime.dotnet.gc.count.gen2",
attribute: "generation",
attributeValue: "gen2",
metricType: pmetric.MetricTypeSum,
}},
}

var javaRuntimeMetricsMappings = map[string][]runtimeMetricMapping{
"process.runtime.jvm.threads.count": {{mappedName: "jvm.thread_count"}},
liustanley marked this conversation as resolved.
Show resolved Hide resolved
"process.runtime.jvm.gc.duration": {{mappedName: "jvm.gc.parnew.time"}},
"process.runtime.jvm.memory.usage": {{
mappedName: "jvm.heap_memory",
attribute: "type",
attributeValue: "heap",
}, {
mappedName: "jvm.non_heap_memory",
attribute: "type",
attributeValue: "non_heap",
}},
"process.runtime.jvm.memory.committed": {{
mappedName: "jvm.heap_memory_committed",
attribute: "type",
attributeValue: "heap",
}, {
mappedName: "jvm.non_heap_memory_committed",
attribute: "type",
attributeValue: "non_heap",
}},
"process.runtime.jvm.memory.init": {{
mappedName: "jvm.heap_memory_init",
attribute: "type",
attributeValue: "heap",
}, {
mappedName: "jvm.non_heap_memory_init",
attribute: "type",
attributeValue: "non_heap",
}},
"process.runtime.jvm.memory.limit": {{
mappedName: "jvm.heap_memory_max",
attribute: "type",
attributeValue: "heap",
}, {
mappedName: "jvm.non_heap_memory_max",
attribute: "type",
attributeValue: "non_heap",
}},
}

func getRuntimeMetricsMappings() map[string][]runtimeMetricMapping {
res := map[string][]runtimeMetricMapping{}
for k, v := range goRuntimeMetricsMappings {
res[k] = v
}
for k, v := range dotnetRuntimeMetricsMappings {
res[k] = v
}
for k, v := range javaRuntimeMetricsMappings {
res[k] = v
}
return res
}

// runtimeMetricsMappings defines the mappings from OTel runtime metric names to their
// equivalent Datadog runtime metric names
var runtimeMetricsMappings = getRuntimeMetricsMappings()

const metricName string = "metric name"

var _ source.Provider = (*noSourceProvider)(nil)
Expand Down Expand Up @@ -586,9 +638,9 @@ func (t *Translator) MapMetrics(ctx context.Context, md pmetric.Metrics, consume
cp.SetName(mp.mappedName)
break
}
if mp.metricType == pmetric.MetricTypeSum {
if md.Type() == pmetric.MetricTypeSum {
mapSumRuntimeMetricWithAttributes(md, metricsArray, mp)
} else if mp.metricType == pmetric.MetricTypeGauge {
} else if md.Type() == pmetric.MetricTypeGauge {
mapGaugeRuntimeMetricWithAttributes(md, metricsArray, mp)
}
}
Expand Down
36 changes: 30 additions & 6 deletions pkg/otlp/metrics/metrics_translator_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,6 @@ package metrics

import (
"context"
"fmt"
"math"
"testing"
"time"
Expand Down Expand Up @@ -428,7 +427,7 @@ func TestMapSumRuntimeMetricWithAttributesHasMapping(t *testing.T) {
ctx := context.Background()
tr := newTranslator(t, zap.NewNop())
consumer := &mockFullConsumer{}
if err := tr.MapMetrics(ctx, createTestMetricWithAttributes(false, "process.runtime.dotnet.gc.collections.count", pmetric.MetricTypeSum, "generation", "gen"), consumer); err != nil {
if err := tr.MapMetrics(ctx, createTestMetricWithAttributes(false, "process.runtime.dotnet.gc.collections.count", pmetric.MetricTypeSum, "generation", []string{"gen0", "gen1", "gen2"}), consumer); err != nil {
t.Fatal(err)
}
startTs := int(getProcessStartTime()) + 1
Expand All @@ -447,7 +446,7 @@ func TestMapGaugeRuntimeMetricWithAttributesHasMapping(t *testing.T) {
ctx := context.Background()
tr := newTranslator(t, zap.NewNop())
consumer := &mockFullConsumer{}
if err := tr.MapMetrics(ctx, createTestMetricWithAttributes(false, "process.runtime.dotnet.gc.heap.size", pmetric.MetricTypeGauge, "generation", "gen"), consumer); err != nil {
if err := tr.MapMetrics(ctx, createTestMetricWithAttributes(false, "process.runtime.dotnet.gc.heap.size", pmetric.MetricTypeGauge, "generation", []string{"gen0", "gen1", "gen2"}), consumer); err != nil {
t.Fatal(err)
}
startTs := int(getProcessStartTime()) + 1
Expand All @@ -464,6 +463,27 @@ func TestMapGaugeRuntimeMetricWithAttributesHasMapping(t *testing.T) {
)
}

func TestMapGaugeRuntimeMetricWithInvalidAttributes(t *testing.T) {
ctx := context.Background()
tr := newTranslator(t, zap.NewNop())
consumer := &mockFullConsumer{}
if err := tr.MapMetrics(ctx, createTestMetricWithAttributes(false, "process.runtime.jvm.memory.usage", pmetric.MetricTypeGauge, "type", []string{"heap", "heap1", "heap2", "non_heap"}), consumer); err != nil {
t.Fatal(err)
}
startTs := int(getProcessStartTime()) + 1
assert.ElementsMatch(t,
consumer.metrics,
[]metric{
newGaugeWithHost(newDims("process.runtime.jvm.memory.usage").AddTags("type:heap"), uint64(seconds(startTs+1)), 10, fallbackHostname),
newGaugeWithHost(newDims("process.runtime.jvm.memory.usage").AddTags("type:heap1"), uint64(seconds(startTs+2)), 15, fallbackHostname),
newGaugeWithHost(newDims("process.runtime.jvm.memory.usage").AddTags("type:heap2"), uint64(seconds(startTs+3)), 20, fallbackHostname),
newGaugeWithHost(newDims("process.runtime.jvm.memory.usage").AddTags("type:non_heap"), uint64(seconds(startTs+4)), 25, fallbackHostname),
newGaugeWithHost(newDims("jvm.heap_memory").AddTags("type:heap"), uint64(seconds(startTs+1)), 10, fallbackHostname),
newGaugeWithHost(newDims("jvm.non_heap_memory").AddTags("type:non_heap"), uint64(seconds(startTs+4)), 25, fallbackHostname),
},
)
}

func TestMapRuntimeMetricsNoMapping(t *testing.T) {
ctx := context.Background()
tr := newTranslator(t, zap.NewNop())
Expand Down Expand Up @@ -827,7 +847,7 @@ func createTestDoubleCumulativeMonotonicMetrics(tsmatch bool) pmetric.Metrics {
return md
}

func createTestMetricWithAttributes(tsmatch bool, metricName string, metricType pmetric.MetricType, attribute string, attributeValue string) pmetric.Metrics {
func createTestMetricWithAttributes(tsmatch bool, metricName string, metricType pmetric.MetricType, attribute string, attributeValues []string) pmetric.Metrics {
md := pmetric.NewMetrics()
met := md.ResourceMetrics().AppendEmpty().ScopeMetrics().AppendEmpty().Metrics().AppendEmpty()
met.SetName(metricName)
Expand All @@ -841,12 +861,16 @@ func createTestMetricWithAttributes(tsmatch bool, metricName string, metricType
met.SetEmptyGauge()
dpsInt = met.Gauge().DataPoints()
}
values := []int64{10, 15, 20}

var values []int64
for i := range attributeValues {
values = append(values, int64(10+(i*5)))
}
dpsInt.EnsureCapacity(len(values))
startTs := int(getProcessStartTime()) + 1
for i, val := range values {
dpInt := dpsInt.AppendEmpty()
dpInt.Attributes().PutStr(attribute, fmt.Sprintf("%v%v", attributeValue, i))
dpInt.Attributes().PutStr(attribute, attributeValues[i])
dpInt.SetStartTimestamp(seconds(startTs))
if tsmatch {
dpInt.SetTimestamp(seconds(startTs))
Expand Down