[internal/datadog] Fix Datarace in metrics client (#31964)

**Description:** <Describe what has changed.>
<!--Ex. Fixing a bug - Describe the bug and how this fixes the issue.
Ex. Adding a feature - Explain what this achieves.-->
Fixes the data race.
```
fatal error: concurrent map read and map write

goroutine 72665 [running]:
github.com/open-telemetry/opentelemetry-collector-contrib/internal/datadog.(*metricsClient).Gauge.func1({0x7b895a0?, 0xc07c930790?}, {0x940dac8, 0xc1996ce180})
	github.com/open-telemetry/opentelemetry-collector-contrib/internal/[email protected]/metrics_client.go:49 +0x85
go.opentelemetry.io/otel/sdk/metric.(*meter).float64ObservableInstrument.func1.1({0x9443f08, 0xef10b00})
	go.opentelemetry.io/otel/sdk/[email protected]/meter.go:286 +0x55
go.opentelemetry.io/otel/sdk/metric.(*pipeline).produce(0xc002852bd0, {0x9443f08, 0xef10b00}, 0xc108dd46c0)
	go.opentelemetry.io/otel/sdk/[email protected]/pipeline.go:122 +0x168
go.opentelemetry.io/otel/sdk/metric.(*ManualReader).Collect(0xc0030d2aa0, {0x9443f08, 0xef10b00}, 0xc108dd46c0)
	go.opentelemetry.io/otel/sdk/[email protected]/manual_reader.go:123 +0xe2
go.opentelemetry.io/otel/exporters/prometheus.(*collector).Collect(0xc0030e4aa0, 0xc10a0083c0)
	go.opentelemetry.io/otel/exporters/[email protected]/exporter.go:158 +0x72
github.com/prometheus/client_golang/prometheus.(*Registry).Gather.func1()
	github.com/prometheus/[email protected]/prometheus/registry.go:457 +0xe5
created by github.com/prometheus/client_golang/prometheus.(*Registry).Gather in goroutine 5
```

**Link to tracking Issue:** <Issue number if applicable>

**Testing:** <Describe what testing was performed and which tests were
added.>

**Documentation:** <Describe the documentation added.>

---------

Co-authored-by: Yang Song <[email protected]>

.chloggen/dinesh.gurumurthy_fix-data-race-metrics-client.yaml

@@ -0,0 +1,22 @@
+# Use this changelog template to create an entry for release notes.
+
+# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix'
+change_type: bug_fix
+# The name of the component, or a single word describing the area of concern, (e.g. filelogreceiver)
+component: datadog/connector
+# A brief description of the change.  Surround your text with quotes ("") if it needs to start with a backtick (`).
+note: "Fix data race in datadog metrics client"
+# Mandatory: One or more tracking issues related to the change. You can use the PR number here if no issue exists.
+issues: [31964]
+# (Optional) One or more lines of additional information to render under the primary note.
+# These lines will be padded with 2 spaces and then inserted directly into the document.
+# Use pipe (|) for multiline entries.
+subtext: "The PR ensures mutex protects gauges map in every code path."
+# If your change doesn't affect end users or the exported elements of any package,
+# you should instead start your pull request title with [chore] or use the "Skip Changelog" label.
+# Optional: The change log or logs in which this entry should be included.
+# e.g. '[user]' or '[user, api]'
+# Include 'user' if the change is relevant to end users.
+# Include 'api' if there is a change to a library API.
+# Default: '[user]'
+change_logs: []

internal/datadog/metrics_client.go

@@ -46,6 +46,8 @@ func (m *metricsClient) Gauge(name string, value float64, tags []string, _ float
 	m.gauges[name] = value
 	_, err := m.meter.Float64ObservableGauge(name, metric.WithFloat64Callback(func(_ context.Context, f metric.Float64Observer) error {
 		attr := m.attributeFromTags(tags)
+		m.mutex.Lock()
+		defer m.mutex.Unlock()
 		if v, ok := m.gauges[name]; ok {
 			f.Observe(v, metric.WithAttributeSet(attr))
 		}

internal/datadog/metrics_client_test.go

@@ -4,6 +4,7 @@ package datadog
 
 import (
 	"context"
+	"sync"
 	"testing"
 	"time"
 
@@ -72,6 +73,43 @@ func TestGaugeMultiple(t *testing.T) {
 	metricdatatest.AssertEqual(t, want, got, metricdatatest.IgnoreTimestamp())
 }
 
+func TestGaugeDataRace(t *testing.T) {
+	reader, metricClient, _ := setupMetricClient()
+	var wg sync.WaitGroup
+	wg.Add(2)
+
+	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+	defer cancel()
+
+	go func() {
+		defer wg.Done()
+		for {
+			select {
+			case <-ctx.Done():
+				return
+			default:
+				err := metricClient.Gauge("test_gauge", 1, []string{"otlp:true"}, 1)
+				assert.NoError(t, err)
+			}
+		}
+	}()
+
+	go func() {
+		defer wg.Done()
+		for {
+			select {
+			case <-ctx.Done():
+				return
+			default:
+				err := reader.Collect(context.Background(), &metricdata.ResourceMetrics{})
+				assert.NoError(t, err)
+			}
+		}
+	}()
+
+	wg.Wait()
+}
+
 func TestCount(t *testing.T) {
 	reader, metricClient, _ := setupMetricClient()