[receiver/prometheusreceiver] Fix retrieval of aggregated metrics wit…

…h no job/instance label (#33565)

**Description:** This PR fixes the retrieval of metrics where either the
`job` or `instance` label is missing, and `honor_labels` is set to
`true`. This can be the case for aggregated metrics coming from a
federate endpoint. This PR introduces a fallback to using the
`job`/`instance` labels from the scrape config for such metrics.

**Link to tracking Issue:** Fixes #32555

**Testing:** 
- Added a Unit test
- Verified using the following config:

```
receivers:
  prometheus:
    config:
      scrape_configs:
        - job_name: 'federate'
          scrape_interval: 10s
          honor_labels: true
          params:
            'match[]':
              - '{__name__="cluster:node_cpu:sum_rate5m"}'
          metrics_path: '/federate'
          static_configs:
            - targets:
                - "localhost:9090"
exporters:
  debug:
    verbosity: detailed
  otlphttp:
    endpoint: ${env:OTLP_ENDPOINT}

  pipelines:
    metrics:
      receivers: [otlp,prometheus]
      exporters: [otlphttp, debug]
```

This was tested on a `kind` K8s cluster running the prometheus operator,
with a port forward for the `prometheus-k8s` service created by the
prometheus operator (therefore the `localhost:9090` address in the
target).

---------

Signed-off-by: Florian Bacher <[email protected]>
Co-authored-by: David Ashpole <[email protected]>

.chloggen/fix-prometheusreceiver-federate-endpoint.yaml

@@ -0,0 +1,27 @@
+# Use this changelog template to create an entry for release notes.
+
+# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix'
+change_type: bug_fix
+
+# The name of the component, or a single word describing the area of concern, (e.g. filelogreceiver)
+component: prometheusreceiver
+
+# A brief description of the change.  Surround your text with quotes ("") if it needs to start with a backtick (`).
+note: Fall back to scrape config job/instance labels for aggregated metrics without instance/job labels
+
+# Mandatory: One or more tracking issues related to the change. You can use the PR number here if no issue exists.
+issues: [32555]
+
+# (Optional) One or more lines of additional information to render under the primary note.
+# These lines will be padded with 2 spaces and then inserted directly into the document.
+# Use pipe (|) for multiline entries.
+subtext:
+
+# If your change doesn't affect end users or the exported elements of any package,
+# you should instead start your pull request title with [chore] or use the "Skip Changelog" label.
+# Optional: The change log or logs in which this entry should be included.
+# e.g. '[user]' or '[user, api]'
+# Include 'user' if the change is relevant to end users.
+# Include 'api' if there is a change to a library API.
+# Default: '[user]'
+change_logs: []

receiver/prometheusreceiver/internal/transaction.go

@@ -365,15 +365,41 @@ func (t *transaction) initTransaction(labels labels.Labels) error {
 		return errors.New("unable to find MetricMetadataStore in context")
 	}
 
-	job, instance := labels.Get(model.JobLabel), labels.Get(model.InstanceLabel)
-	if job == "" || instance == "" {
-		return errNoJobInstance
+	job, instance, err := t.getJobAndInstance(labels)
+	if err != nil {
+		return err
 	}
 	t.nodeResource = CreateResource(job, instance, target.DiscoveredLabels())
 	t.isNew = false
 	return nil
 }
 
+func (t *transaction) getJobAndInstance(labels labels.Labels) (string, string, error) {
+	// first, try to get job and instance from the labels
+	job, instance := labels.Get(model.JobLabel), labels.Get(model.InstanceLabel)
+	if job != "" && instance != "" {
+		return job, instance, nil
+	}
+
+	// if not available in the labels, try to fall back to the scrape job associated
+	// with the transaction.
+	// this can be the case for, e.g., aggregated metrics coming from a federate endpoint
+	// that represent the whole cluster, rather than an individual workload.
+	// See https://github.com/open-telemetry/opentelemetry-collector-contrib/issues/32555 for reference
+	if target, ok := scrape.TargetFromContext(t.ctx); ok {
+		if job == "" {
+			job = target.GetValue(model.JobLabel)
+		}
+		if instance == "" {
+			instance = target.GetValue(model.InstanceLabel)
+		}
+		if job != "" && instance != "" {
+			return job, instance, nil
+		}
+	}
+	return "", "", errNoJobInstance
+}
+
 func (t *transaction) Commit() error {
 	if t.isNew {
 		return nil

receiver/prometheusreceiver/internal/transaction_test.go

@@ -389,6 +389,42 @@ func testTransactionAppendValidAndInvalid(t *testing.T, enableNativeHistograms b
 	require.Equal(t, 1, mds[0].MetricCount())
 }
 
+func TestTransactionAppendWithEmptyLabelArrayFallbackToTargetLabels(t *testing.T) {
+	for _, enableNativeHistograms := range []bool{true, false} {
+		t.Run(fmt.Sprintf("enableNativeHistograms=%v", enableNativeHistograms), func(t *testing.T) {
+			testTransactionAppendWithEmptyLabelArrayFallbackToTargetLabels(t, enableNativeHistograms)
+		})
+	}
+}
+
+func testTransactionAppendWithEmptyLabelArrayFallbackToTargetLabels(t *testing.T, enableNativeHistograms bool) {
+	sink := new(consumertest.MetricsSink)
+
+	scrapeTarget := scrape.NewTarget(
+		// processedLabels contain label values after processing (e.g. relabeling)
+		labels.FromMap(map[string]string{
+			model.InstanceLabel: "localhost:8080",
+			model.JobLabel:      "federate",
+		}),
+		// discoveredLabels contain labels prior to any processing
+		labels.FromMap(map[string]string{
+			model.AddressLabel: "address:8080",
+			model.SchemeLabel:  "http",
+		}),
+		nil)
+
+	ctx := scrape.ContextWithMetricMetadataStore(
+		scrape.ContextWithTarget(context.Background(), scrapeTarget),
+		testMetadataStore(testMetadata))
+
+	tr := newTransaction(ctx, &startTimeAdjuster{startTime: startTimestamp}, sink, labels.EmptyLabels(), receivertest.NewNopSettings(), nopObsRecv(t), false, enableNativeHistograms)
+
+	_, err := tr.Append(0, labels.FromMap(map[string]string{
+		model.MetricNameLabel: "counter_test",
+	}), time.Now().Unix()*1000, 1.0)
+	assert.NoError(t, err)
+}
+
 func TestAppendExemplarWithNoMetricName(t *testing.T) {
 	for _, enableNativeHistograms := range []bool{true, false} {
 		t.Run(fmt.Sprintf("enableNativeHistograms=%v", enableNativeHistograms), func(t *testing.T) {