[pkg/otlp/metrics] Add telemetry metric counting the number of missing sources

.chloggen/mx-psi_hostname-counter.yaml

@@ -0,0 +1,16 @@
+# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix'
+change_type: enhancement
+
+# The name of the component (e.g. pkg/quantile)
+component: pkg/otlp/metric
+
+# A brief description of the change. Surround your text with quotes ("") if it needs to start with a backtick (`).
+note: Add `datadog.otlp_translator.metrics.missing_source` counter, which counts the number of metrics that are missing a source (e.g. hostname).
+
+# The PR related to this change
+issues: [220]
+
+# (Optional) One or more lines of additional information to render under the primary note.
+# These lines will be padded with 2 spaces and then inserted directly into the document.
+# Use pipe (|) for multiline entries.
+subtext:

.copyright-overrides.yml

@@ -13,3 +13,5 @@ google.golang.org/genproto/*: Copyright 2020 Google LLC
 github.com/DataDog/opentelemetry-mapping-go/*: Copyright [2023-Present] Datadog, Inc.
 github.com/modern-go/concurrent: Copyright (c) 2018 Tao Wen
 github.com/modern-go/reflect2: Copyright (c) 2018 Tao Wen
+github.com/go-logr/logr: "Copyright 2019 The logr Authors."
+github.com/go-logr/stdr: "Copyright 2019 The logr Authors."

LICENSE-3rdparty.csv

@@ -161,6 +161,9 @@ pkg/otlp/metrics,github.com/DataDog/sketches-go/ddsketch/pb/sketchpb,Apache-2.0,
 pkg/otlp/metrics,github.com/DataDog/sketches-go/ddsketch/stat,Apache-2.0,"Copyright 2021 DataDog, Inc | Copyright 2021 Datadog, Inc"
 pkg/otlp/metrics,github.com/DataDog/sketches-go/ddsketch/store,Apache-2.0,"Copyright 2021 DataDog, Inc | Copyright 2021 Datadog, Inc"
 pkg/otlp/metrics,github.com/dustin/go-humanize,MIT,Copyright (c) 2005-2008  Dustin Sallings <[email protected]>
+pkg/otlp/metrics,github.com/go-logr/logr,Apache-2.0,Copyright 2019 The logr Authors.
+pkg/otlp/metrics,github.com/go-logr/logr/funcr,Apache-2.0,Copyright 2019 The logr Authors.
+pkg/otlp/metrics,github.com/go-logr/stdr,Apache-2.0,Copyright 2019 The logr Authors.
 pkg/otlp/metrics,github.com/gogo/protobuf/gogoproto,BSD-3-Clause,"Copyright (c) 2013, The GoGo Authors. All rights reserved | Copyright 2010 The Go Authors.  All rights reserved | Sendgrid, Inc | Vastech SA (PTY) LTD | Walter Schulze <[email protected]> | Anton Povarov <[email protected]> | Brian Goff <[email protected]> | Clayton Coleman <[email protected]> | Denis Smirnov <[email protected]> | DongYun Kang <[email protected]> | Dwayne Schultz <[email protected]> | Georg Apitz <[email protected]> | Gustav Paul <[email protected]> | Johan Brandhorst <[email protected]> | John Shahid <[email protected]> | John Tuley <[email protected]> | Laurent <[email protected]> | Patrick Lee <[email protected]> | Peter Edge <[email protected]> | Roger Johansson <[email protected]> | Sam Nguyen <[email protected]> | Sergio Arbeo <[email protected]> | Stephen J Day <[email protected]> | Tamir Duberstein <[email protected]> | Todd Eisenberger <[email protected]> | Tormod Erevik Lea <[email protected]> | Vyacheslav Kim <[email protected]> | Walter Schulze <[email protected]>"
 pkg/otlp/metrics,github.com/gogo/protobuf/jsonpb,BSD-3-Clause,"Copyright (c) 2013, The GoGo Authors. All rights reserved | Copyright 2010 The Go Authors.  All rights reserved | Sendgrid, Inc | Vastech SA (PTY) LTD | Walter Schulze <[email protected]> | Anton Povarov <[email protected]> | Brian Goff <[email protected]> | Clayton Coleman <[email protected]> | Denis Smirnov <[email protected]> | DongYun Kang <[email protected]> | Dwayne Schultz <[email protected]> | Georg Apitz <[email protected]> | Gustav Paul <[email protected]> | Johan Brandhorst <[email protected]> | John Shahid <[email protected]> | John Tuley <[email protected]> | Laurent <[email protected]> | Patrick Lee <[email protected]> | Peter Edge <[email protected]> | Roger Johansson <[email protected]> | Sam Nguyen <[email protected]> | Sergio Arbeo <[email protected]> | Stephen J Day <[email protected]> | Tamir Duberstein <[email protected]> | Todd Eisenberger <[email protected]> | Tormod Erevik Lea <[email protected]> | Vyacheslav Kim <[email protected]> | Walter Schulze <[email protected]>"
 pkg/otlp/metrics,github.com/gogo/protobuf/proto,BSD-3-Clause,"Copyright (c) 2013, The GoGo Authors. All rights reserved | Copyright 2010 The Go Authors.  All rights reserved | Sendgrid, Inc | Vastech SA (PTY) LTD | Walter Schulze <[email protected]> | Anton Povarov <[email protected]> | Brian Goff <[email protected]> | Clayton Coleman <[email protected]> | Denis Smirnov <[email protected]> | DongYun Kang <[email protected]> | Dwayne Schultz <[email protected]> | Georg Apitz <[email protected]> | Gustav Paul <[email protected]> | Johan Brandhorst <[email protected]> | John Shahid <[email protected]> | John Tuley <[email protected]> | Laurent <[email protected]> | Patrick Lee <[email protected]> | Peter Edge <[email protected]> | Roger Johansson <[email protected]> | Sam Nguyen <[email protected]> | Sergio Arbeo <[email protected]> | Stephen J Day <[email protected]> | Tamir Duberstein <[email protected]> | Todd Eisenberger <[email protected]> | Tormod Erevik Lea <[email protected]> | Vyacheslav Kim <[email protected]> | Walter Schulze <[email protected]>"
@@ -207,12 +210,15 @@ pkg/otlp/metrics,go.opentelemetry.io/collector/pdata/pcommon,Apache-2.0,Copyrigh
 pkg/otlp/metrics,go.opentelemetry.io/collector/pdata/pmetric,Apache-2.0,Copyright The OpenTelemetry Authors
 pkg/otlp/metrics,go.opentelemetry.io/collector/semconv/v1.6.1,Apache-2.0,Copyright The OpenTelemetry Authors
 pkg/otlp/metrics,go.opentelemetry.io/otel/attribute,Apache-2.0,Copyright The OpenTelemetry Authors
+pkg/otlp/metrics,go.opentelemetry.io/otel/baggage,Apache-2.0,Copyright The OpenTelemetry Authors
 pkg/otlp/metrics,go.opentelemetry.io/otel/codes,Apache-2.0,Copyright The OpenTelemetry Authors
 pkg/otlp/metrics,go.opentelemetry.io/otel/internal,Apache-2.0,Copyright The OpenTelemetry Authors
 pkg/otlp/metrics,go.opentelemetry.io/otel/internal/attribute,Apache-2.0,Copyright The OpenTelemetry Authors
+pkg/otlp/metrics,go.opentelemetry.io/otel/internal/baggage,Apache-2.0,Copyright The OpenTelemetry Authors
 pkg/otlp/metrics,go.opentelemetry.io/otel/metric,Apache-2.0,Copyright The OpenTelemetry Authors
 pkg/otlp/metrics,go.opentelemetry.io/otel/metric/embedded,Apache-2.0,Copyright The OpenTelemetry Authors
 pkg/otlp/metrics,go.opentelemetry.io/otel/metric/noop,Apache-2.0,Copyright The OpenTelemetry Authors
+pkg/otlp/metrics,go.opentelemetry.io/otel/propagation,Apache-2.0,Copyright The OpenTelemetry Authors
 pkg/otlp/metrics,go.opentelemetry.io/otel/trace,Apache-2.0,Copyright The OpenTelemetry Authors
 pkg/otlp/metrics,go.opentelemetry.io/otel/trace/embedded,Apache-2.0,Copyright The OpenTelemetry Authors
 pkg/otlp/metrics,go.opentelemetry.io/otel/trace/noop,Apache-2.0,Copyright The OpenTelemetry Authors

pkg/otlp/metrics/go.mod

@@ -14,13 +14,17 @@ require (
 	github.com/stretchr/testify v1.8.4
 	go.opentelemetry.io/collector/component v0.91.0
 	go.opentelemetry.io/collector/pdata v1.0.0
+	go.opentelemetry.io/otel/metric v1.21.0
+	go.opentelemetry.io/otel/sdk/metric v1.21.0
 	go.uber.org/zap v1.26.0
 	golang.org/x/exp v0.0.0-20230321023759-10a507213a29
 )
 
 require (
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/dustin/go-humanize v1.0.1 // indirect
+	github.com/go-logr/logr v1.3.0 // indirect
+	github.com/go-logr/stdr v1.2.2 // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
 	github.com/hashicorp/go-version v1.6.0 // indirect
 	github.com/json-iterator/go v1.1.12 // indirect
@@ -40,7 +44,7 @@ require (
 	go.opentelemetry.io/collector/featuregate v1.0.0 // indirect
 	go.opentelemetry.io/collector/semconv v0.90.1 // indirect
 	go.opentelemetry.io/otel v1.21.0 // indirect
-	go.opentelemetry.io/otel/metric v1.21.0 // indirect
+	go.opentelemetry.io/otel/sdk v1.21.0 // indirect
 	go.opentelemetry.io/otel/trace v1.21.0 // indirect
 	go.uber.org/multierr v1.11.0 // indirect
 	golang.org/x/net v0.18.0 // indirect

pkg/otlp/metrics/go.sum

@@ -7,8 +7,11 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
 github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
+github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
 github.com/go-logr/logr v1.3.0 h1:2y3SDp0ZXuc6/cjLSZ+Q3ir+QB9T/iG5yYRXqsagWSY=
+github.com/go-logr/logr v1.3.0/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
 github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag=
+github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE=
 github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=
 github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
 github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
@@ -81,6 +84,10 @@ go.opentelemetry.io/otel v1.21.0 h1:hzLeKBZEL7Okw2mGzZ0cc4k/A7Fta0uoPgaJCr8fsFc=
 go.opentelemetry.io/otel v1.21.0/go.mod h1:QZzNPQPm1zLX4gZK4cMi+71eaorMSGT3A4znnUvNNEo=
 go.opentelemetry.io/otel/metric v1.21.0 h1:tlYWfeo+Bocx5kLEloTjbcDwBuELRrIFxwdQ36PlJu4=
 go.opentelemetry.io/otel/metric v1.21.0/go.mod h1:o1p3CA8nNHW8j5yuQLdc1eeqEaPfzug24uvsyIEJRWM=
+go.opentelemetry.io/otel/sdk v1.21.0 h1:FTt8qirL1EysG6sTQRZ5TokkU8d0ugCj8htOgThZXQ8=
+go.opentelemetry.io/otel/sdk v1.21.0/go.mod h1:Nna6Yv7PWTdgJHVRD9hIYywQBRx7pbox6nwBnZIxl/E=
+go.opentelemetry.io/otel/sdk/metric v1.21.0 h1:smhI5oD714d6jHE6Tie36fPx4WDFIg+Y6RfAY4ICcR0=
+go.opentelemetry.io/otel/sdk/metric v1.21.0/go.mod h1:FJ8RAsoPGv/wYMgBdUJXOm+6pzFY3YdljnXtv1SBE8Q=
 go.opentelemetry.io/otel/trace v1.21.0 h1:WD9i5gzvoUPuXIXH24ZNBudiarZDKuekPqi/E8fpfLc=
 go.opentelemetry.io/otel/trace v1.21.0/go.mod h1:LGbsEB0f9LGjN+OZaQQ26sohbOmiMR+BaslueVtS/qQ=
 go.uber.org/goleak v1.2.0 h1:xqgm/S+aQvhWFTtR0XK3Jvg7z8kGV8P4X14IzwN3Eqk=

pkg/otlp/metrics/metrics_translator.go

@@ -26,6 +26,7 @@ import (
 	"go.opentelemetry.io/collector/component"
 	"go.opentelemetry.io/collector/pdata/pcommon"
 	"go.opentelemetry.io/collector/pdata/pmetric"
+	otelmetric "go.opentelemetry.io/otel/metric"
 	"go.uber.org/zap"
 	"golang.org/x/exp/slices"
 
@@ -39,6 +40,9 @@ import (
 const (
 	metricName             string = "metric name"
 	errNoBucketsNoSumCount string = "no buckets mode and no send count sum are incompatible"
+
+	meterName               string = "github.com/DataDog/opentelemetry-mapping-go/pkg/otlp/metrics"
+	missingSourceMetricName string = "datadog.otlp_translator.metrics.missing_source"
 )
 
 var _ source.Provider = (*noSourceProvider)(nil)
@@ -51,9 +55,10 @@ func (*noSourceProvider) Source(context.Context) (source.Source, error) {
 
 // Translator is a metrics translator.
 type Translator struct {
-	prevPts *ttlCache
-	logger  *zap.Logger
-	cfg     translatorConfig
+	prevPts        *ttlCache
+	logger         *zap.Logger
+	missingSources otelmetric.Int64Counter
+	cfg            translatorConfig
 }
 
 // Metadata specifies information about the outcome of the MapMetrics call.
@@ -88,10 +93,21 @@ func NewTranslator(set component.TelemetrySettings, options ...TranslatorOption)
 	}
 
 	cache := newTTLCache(cfg.sweepInterval, cfg.deltaTTL)
+	meter := set.MeterProvider.Meter(meterName)
+	missingSources, err := meter.Int64Counter(
+		missingSourceMetricName,
+		otelmetric.WithDescription("OTLP metrics that are missing a source (e.g. hostname)"),
+		otelmetric.WithUnit("[metric]"),
+	)
+	if err != nil {
+		return nil, fmt.Errorf("failed to build missing source counter: %w", err)
+	}
+
 	return &Translator{
-		prevPts: cache,
-		logger:  set.Logger.With(zap.String("component", "metrics translator")),
-		cfg:     cfg,
+		prevPts:        cache,
+		logger:         set.Logger.With(zap.String("component", "metrics translator")),
+		missingSources: missingSources,
+		cfg:            cfg,
 	}, nil
 }
 
@@ -524,16 +540,16 @@ func (t *Translator) mapSummaryMetrics(
 	}
 }
 
-func (t *Translator) source(m pcommon.Map) (source.Source, error) {
-	src, ok := attributes.SourceFromAttrs(m)
-	if !ok {
+func (t *Translator) source(ctx context.Context, m pcommon.Map) (source.Source, bool, error) {
+	src, hasSource := attributes.SourceFromAttrs(m)
+	if !hasSource {
 		var err error
-		src, err = t.cfg.fallbackSourceProvider.Source(context.Background())
+		src, err = t.cfg.fallbackSourceProvider.Source(ctx)
 		if err != nil {
-			return source.Source{}, fmt.Errorf("failed to get fallback source: %w", err)
+			return source.Source{}, false, fmt.Errorf("failed to get fallback source: %w", err)
 		}
 	}
-	return src, nil
+	return src, hasSource, nil
 }
 
 // extractLanguageTag appends a new language tag to languageTags if a new language tag is found from the given name
@@ -654,7 +670,7 @@ func (t *Translator) MapMetrics(ctx context.Context, md pmetric.Metrics, consume
 			consumer.ConsumeAPMStats(sp)
 			continue
 		}
-		src, err := t.source(rm.Resource().Attributes())
+		src, hasSource, err := t.source(ctx, rm.Resource().Attributes())
 		if err != nil {
 			return metadata, err
 		}
@@ -679,6 +695,11 @@ func (t *Translator) MapMetrics(ctx context.Context, md pmetric.Metrics, consume
 			ilm := ilms.At(j)
 			metricsArray := ilm.Metrics()
 
+			if !hasSource && ilm.Scope().Name() != meterName {
+				// Only count metrics if they do not come from the translator itself.
+				t.missingSources.Add(ctx, int64(metricsArray.Len()))
+			}
+
 			var additionalTags []string
 			if t.cfg.InstrumentationScopeMetadataAsTags {
 				additionalTags = append(attributeTags, instrumentationscope.TagsFromInstrumentationScopeMetadata(ilm.Scope())...)

pkg/otlp/metrics/source_test.go

@@ -0,0 +1,84 @@
+// Unless explicitly stated otherwise all files in this repository are licensed
+// under the Apache License Version 2.0.
+// This product includes software developed at Datadog (https://www.datadoghq.com/).
+// Copyright 2022-present Datadog, Inc.
+
+package metrics
+
+import (
+	"context"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"go.opentelemetry.io/collector/component/componenttest"
+	sdkmetric "go.opentelemetry.io/otel/sdk/metric"
+	"go.opentelemetry.io/otel/sdk/metric/metricdata"
+)
+
+// deltaSelector sets delta aggregation temporality for monotonic counters and histograms.
+func deltaSelector(kind sdkmetric.InstrumentKind) metricdata.Temporality {
+	switch kind {
+	case sdkmetric.InstrumentKindCounter,
+		sdkmetric.InstrumentKindHistogram,
+		sdkmetric.InstrumentKindObservableGauge,
+		sdkmetric.InstrumentKindObservableCounter:
+		return metricdata.DeltaTemporality
+	case sdkmetric.InstrumentKindUpDownCounter,
+		sdkmetric.InstrumentKindObservableUpDownCounter:
+		return metricdata.CumulativeTemporality
+	}
+	panic("unknown instrument kind")
+}
+
+// AssertHasSumMetric asserts that an OTLP metrics payload has
+// a single sum metric with a single datapoint and with the given name and value.
+func AssertHasSumMetric[N int64 | float64](t *testing.T, rm *metricdata.ResourceMetrics, name string, value int64) {
+	var found bool
+	for _, scopeMetric := range rm.ScopeMetrics {
+		for _, metric := range scopeMetric.Metrics {
+			if metric.Name == name {
+				if !found {
+					assert.Len(t, metric.Data.(metricdata.Sum[N]).DataPoints, 1)
+					assert.Equal(t, value, metric.Data.(metricdata.Sum[N]).DataPoints[0].Value)
+					found = true
+				} else {
+					assert.Fail(t, "metric %s found more than once", name)
+				}
+			}
+		}
+	}
+
+	assert.True(t, found, "metric %s not found", name)
+}
+
+func TestInternalTelemetryMetrics(t *testing.T) {
+	tests := []struct {
+		name               string
+		otlpfile           string
+		ddogfile           string
+		expectedNumMissing int64
+	}{
+		{
+			name:               "simple",
+			otlpfile:           "testdata/otlpdata/source/simple.json",
+			ddogfile:           "testdata/datadogdata/source/simple.json",
+			expectedNumMissing: 4,
+		},
+	}
+
+	for _, testinstance := range tests {
+		t.Run(testinstance.name, func(t *testing.T) {
+			set := componenttest.NewNopTelemetrySettings()
+			reader := sdkmetric.NewManualReader(sdkmetric.WithTemporalitySelector(deltaSelector))
+			set.MeterProvider = sdkmetric.NewMeterProvider(sdkmetric.WithReader(reader))
+			translator, err := NewTranslator(set)
+			require.NoError(t, err)
+			AssertTranslatorMap(t, translator, testinstance.otlpfile, testinstance.ddogfile)
+
+			rm := &metricdata.ResourceMetrics{}
+			assert.NoError(t, reader.Collect(context.Background(), rm))
+			AssertHasSumMetric[int64](t, rm, missingSourceMetricName, testinstance.expectedNumMissing)
+		})
+	}
+}

pkg/otlp/metrics/statspayload.go

@@ -15,6 +15,7 @@
 package metrics
 
 import (
+	"context"
 	"fmt"
 	"strings"
 
@@ -327,7 +328,7 @@ func (t *Translator) statsPayloadFromMetrics(rmx pmetric.ResourceMetrics) (*pb.C
 	hostname := getStr(attr, statsKeyHostname)
 	tags := strings.Split(getStr(attr, statsKeyTags), ",")
 	if hostname == UnsetHostnamePlaceholder {
-		src, err := t.source(attr)
+		src, _, err := t.source(context.Background(), attr)
 		if err != nil {
 			return &pb.ClientStatsPayload{}, err
 		}

pkg/otlp/metrics/testdata/datadogdata/source/simple.json

@@ -0,0 +1,89 @@
+{
+  "Sketches": null,
+  "TimeSeries": [
+    {
+      "Name": "with-source",
+      "Tags": [],
+      "Host": "res-hostname",
+      "OriginID": "",
+      "OriginProduct": 0,
+      "OriginCategory": 0,
+      "OriginService": 0,
+      "Type": "gauge",
+      "Timestamp": 1667560641226420924,
+      "Value": 1
+    },
+    {
+      "Name": "with-source-2",
+      "Tags": [],
+      "Host": "res-hostname",
+      "OriginID": "",
+      "OriginProduct": 0,
+      "OriginCategory": 0,
+      "OriginService": 0,
+      "Type": "gauge",
+      "Timestamp": 1667560641226420924,
+      "Value": 1
+    },
+    {
+      "Name": "missing-source-1",
+      "Tags": [],
+      "Host": "",
+      "OriginID": "",
+      "OriginProduct": 0,
+      "OriginCategory": 0,
+      "OriginService": 0,
+      "Type": "gauge",
+      "Timestamp": 1667560641226420924,
+      "Value": 1
+    },
+    {
+      "Name": "missing-source-2",
+      "Tags": [],
+      "Host": "",
+      "OriginID": "",
+      "OriginProduct": 0,
+      "OriginCategory": 0,
+      "OriginService": 0,
+      "Type": "gauge",
+      "Timestamp": 1667560641226420924,
+      "Value": 1
+    },
+    {
+      "Name": "missing-source-3",
+      "Tags": [],
+      "Host": "",
+      "OriginID": "",
+      "OriginProduct": 0,
+      "OriginCategory": 0,
+      "OriginService": 0,
+      "Type": "gauge",
+      "Timestamp": 1667560641226420924,
+      "Value": 1
+    },
+    {
+      "Name": "missing-source-4",
+      "Tags": [],
+      "Host": "",
+      "OriginID": "",
+      "OriginProduct": 0,
+      "OriginCategory": 0,
+      "OriginService": 0,
+      "Type": "gauge",
+      "Timestamp": 1667560641226420924,
+      "Value": 1
+    },
+    {
+      "Name": "datadog.otlp_translator.metrics.missing_source",
+      "Tags": [],
+      "Host": "",
+      "OriginID": "",
+      "OriginProduct": 0,
+      "OriginCategory": 0,
+      "OriginService": 0,
+      "Type": "gauge",
+      "Timestamp": 1667560641226420924,
+      "Value": 37
+    }
+  ]
+}