Merge branch 'main' into yaml-hook

.chloggen/codeboten_optional-internal-metric.yaml

@@ -0,0 +1,25 @@
+# Use this changelog template to create an entry for release notes.
+
+# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix'
+change_type: enhancement
+
+# The name of the component, or a single word describing the area of concern, (e.g. otlpreceiver)
+component: mdatagen
+
+# A brief description of the change.  Surround your text with quotes ("") if it needs to start with a backtick (`).
+note: add support for optional internal metrics
+
+# One or more tracking issues or pull requests related to the change
+issues: [10316]
+
+# (Optional) One or more lines of additional information to render under the primary note.
+# These lines will be padded with 2 spaces and then inserted directly into the document.
+# Use pipe (|) for multiline entries.
+subtext:
+
+# Optional: The change log or logs in which this entry should be included.
+# e.g. '[user]' or '[user, api]'
+# Include 'user' if the change is relevant to end users.
+# Include 'api' if there is a change to a library API.
+# Default: '[user]'
+change_logs: []

README.md

@@ -155,7 +155,6 @@ Here is a list of community roles with current and previous members:
 - Triagers ([@open-telemetry/collector-triagers](https://github.com/orgs/open-telemetry/teams/collector-triagers)):
 
   - [Andrzej Stencel](https://github.com/andrzej-stencel), Elastic
-  - [Evan Bradley](https://github.com/evan-bradley), Dynatrace
   - Actively seeking contributors to triage issues
 
 - Emeritus Triagers:
@@ -169,6 +168,7 @@ Here is a list of community roles with current and previous members:
 
    - [Antoine Toulme](https://github.com/atoulme), Splunk
    - [Daniel Jaglowski](https://github.com/djaglowski), observIQ
+   - [Evan Bradley](https://github.com/evan-bradley), Dynatrace
    - [Juraci Paixão Kröhling](https://github.com/jpkrohling), Grafana Labs
    - [Tyler Helmuth](https://github.com/TylerHelmuth), Honeycomb
    - [Yang Song](https://github.com/songy23), Datadog

cmd/mdatagen/go.mod

@@ -52,7 +52,7 @@ require (
 	golang.org/x/sys v0.20.0 // indirect
 	google.golang.org/genproto/googleapis/rpc v0.0.0-20240520151616-dc85e6b867a5 // indirect
 	google.golang.org/grpc v1.64.0 // indirect
-	google.golang.org/protobuf v1.34.1 // indirect
+	google.golang.org/protobuf v1.34.2 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 )
 

cmd/mdatagen/internal/samplereceiver/documentation.md

@@ -133,6 +133,16 @@ Cumulative bytes allocated for heap objects (see 'go doc runtime.MemStats.TotalA
 | ---- | ----------- | ---------- | --------- |
 | By | Sum | Int | true |
 
+### queue_length
+
+This metric is optional and therefore not initialized in NewTelemetryBuilder.
+
+For example this metric only exists if feature A is enabled.
+
+| Unit | Metric Type | Value Type |
+| ---- | ----------- | ---------- |
+| 1 | Gauge | Int |
+
 ### request_duration
 
 Duration of request

cmd/mdatagen/internal/samplereceiver/factory.go

@@ -32,7 +32,7 @@ func createMetrics(ctx context.Context, set receiver.Settings, _ component.Confi
 		return nil, err
 	}
 	telemetryBuilder.BatchSizeTriggerSend.Add(ctx, 1)
-	return nopInstance, nil
+	return nopReceiver{telemetryBuilder: telemetryBuilder}, nil
 }
 
 func createLogs(context.Context, receiver.Settings, component.Config, consumer.Logs) (receiver.Logs, error) {
@@ -44,4 +44,9 @@ var nopInstance = &nopReceiver{}
 type nopReceiver struct {
 	component.StartFunc
 	component.ShutdownFunc
+	telemetryBuilder *metadata.TelemetryBuilder
+}
+
+func (r nopReceiver) initOptionalMetric() {
+	_ = r.telemetryBuilder.InitQueueLength(func() int64 { return 1 })
 }

cmd/mdatagen/internal/samplereceiver/metadata.yaml

@@ -174,3 +174,12 @@ telemetry:
         async: true
         value_type: int
         monotonic: true
+    queue_length:
+      enabled: true
+      description: This metric is optional and therefore not initialized in NewTelemetryBuilder.
+      extended_documentation: For example this metric only exists if feature A is enabled.
+      unit: 1
+      optional: true
+      gauge:
+        async: true
+        value_type: int

cmd/mdatagen/internal/samplereceiver/metrics_test.go

@@ -26,7 +26,7 @@ func TestGeneratedMetrics(t *testing.T) {
 func TestComponentTelemetry(t *testing.T) {
 	tt := setupTestTelemetry()
 	factory := NewFactory()
-	_, err := factory.CreateMetricsReceiver(context.Background(), tt.NewSettings(), componenttest.NewNopHost(), new(consumertest.MetricsSink))
+	receiver, err := factory.CreateMetricsReceiver(context.Background(), tt.NewSettings(), componenttest.NewNopHost(), new(consumertest.MetricsSink))
 	require.NoError(t, err)
 	tt.assertMetrics(t, []metricdata.Metrics{
 		{
@@ -58,5 +58,51 @@ func TestComponentTelemetry(t *testing.T) {
 			},
 		},
 	})
+	rcv, ok := receiver.(nopReceiver)
+	require.True(t, ok)
+	rcv.initOptionalMetric()
+	tt.assertMetrics(t, []metricdata.Metrics{
+		{
+			Name:        "batch_size_trigger_send",
+			Description: "Number of times the batch was sent due to a size trigger",
+			Unit:        "1",
+			Data: metricdata.Sum[int64]{
+				Temporality: metricdata.CumulativeTemporality,
+				IsMonotonic: true,
+				DataPoints: []metricdata.DataPoint[int64]{
+					{
+						Value: 1,
+					},
+				},
+			},
+		},
+		{
+			Name:        "process_runtime_total_alloc_bytes",
+			Description: "Cumulative bytes allocated for heap objects (see 'go doc runtime.MemStats.TotalAlloc')",
+			Unit:        "By",
+			Data: metricdata.Sum[int64]{
+				Temporality: metricdata.CumulativeTemporality,
+				IsMonotonic: true,
+				DataPoints: []metricdata.DataPoint[int64]{
+					{
+						Value: 2,
+					},
+				},
+			},
+		},
+		{
+			Name:        "queue_length",
+			Description: "This metric is optional and therefore not initialized in NewTelemetryBuilder.",
+			Unit:        "1",
+			Data: metricdata.Gauge[int64]{
+				DataPoints: []metricdata.DataPoint[int64]{
+					{
+						Value: 1,
+					},
+				},
+			},
+		},
+	})
 	require.NoError(t, tt.Shutdown(context.Background()))
+
 }

cmd/mdatagen/loader.go

@@ -111,6 +111,10 @@ type metric struct {
 	// be appended to the description used in generated documentation.
 	ExtendedDocumentation string `mapstructure:"extended_documentation"`
 
+	// Optional can be used to specify metrics that may
+	// or may not be present in all cases, depending on configuration.
+	Optional bool `mapstructure:"optional"`
+
 	// Unit of the metric.
 	Unit *string `mapstructure:"unit"`
 

cmd/mdatagen/loader_test.go

@@ -263,6 +263,19 @@ func TestLoadMetadata(t *testing.T) {
 								Async: true,
 							},
 						},
+						"queue_length": {
+							Enabled:               true,
+							Description:           "This metric is optional and therefore not initialized in NewTelemetryBuilder.",
+							ExtendedDocumentation: "For example this metric only exists if feature A is enabled.",
+							Unit:                  strPtr("1"),
+							Optional:              true,
+							Gauge: &gauge{
+								MetricValueType: MetricValueType{
+									ValueType: pmetric.NumberDataPointValueTypeInt,
+								},
+								Async: true,
+							},
+						},
 					},
 				},
 				ScopeName:       "go.opentelemetry.io/collector/internal/receiver/samplereceiver",

cmd/mdatagen/metadata-schema.yaml

@@ -140,6 +140,9 @@ telemetry:
       description:
       # Optional: extended documentation of the metric.
       extended_documentation:
+      # Optional: whether or not this metric is optional. Optional metrics may only be initialized
+      # if certain features are enabled or configured.
+      optional: bool
       # Optional: warnings that will be shown to user under specified conditions.
       warnings:
         # A warning that will be displayed if the metric is enabled in user config.

cmd/mdatagen/templates/component_telemetry_test.go.tmpl

@@ -12,15 +12,18 @@ import (
 	sdkmetric "go.opentelemetry.io/otel/sdk/metric"
 
 	"go.opentelemetry.io/collector/component"
+	{{- if or isConnector isExporter isExtension isProcessor isReceiver }}
 	"go.opentelemetry.io/collector/{{ .Status.Class }}"
 	"go.opentelemetry.io/collector/{{ .Status.Class }}/{{ .Status.Class }}test"
+	{{- end }}
 )
 
 type componentTestTelemetry struct {
 	reader        *sdkmetric.ManualReader
 	meterProvider *sdkmetric.MeterProvider
 }
 
+{{- if or isConnector isExporter isExtension isProcessor isReceiver }}
 func (tt *componentTestTelemetry) NewSettings() {{ .Status.Class }}.Settings {
 	settings := {{ .Status.Class }}test.NewNopSettings()
 	settings.MeterProvider = tt.meterProvider
@@ -29,6 +32,8 @@ func (tt *componentTestTelemetry) NewSettings() {{ .Status.Class }}.Settings {
 	return settings
 }
 
+{{- end }}
+
 func setupTestTelemetry() componentTestTelemetry {
 	reader := sdkmetric.NewManualReader()
 	return componentTestTelemetry{

cmd/mdatagen/templates/telemetry.go.tmpl

@@ -32,7 +32,7 @@ type TelemetryBuilder struct {
     meter metric.Meter
 	{{- range $name, $metric := .Telemetry.Metrics }}
 	{{ $name.Render }} metric.{{ $metric.Data.Instrument }}
-    {{- if $metric.Data.Async }}
+    {{- if and ($metric.Data.Async) (not $metric.Optional) }}
     observe{{ $name.Render }} func() {{ $metric.Data.BasicType }}
     {{- end }}
 	{{- end }}
@@ -60,6 +60,28 @@ func WithAttributeSet(set attribute.Set) telemetryBuilderOption {
 {{- end }}
 
 {{- range $name, $metric := .Telemetry.Metrics }}
+    {{- if $metric.Optional }}
+// Init{{ $name.Render }} configures the {{ $name.Render }} metric.
+func (builder *TelemetryBuilder) Init{{ $name.Render }}({{ if $metric.Data.Async -}}cb func() {{ $metric.Data.BasicType }}{{- end }}) error {
+    var err error
+    builder.{{ $name.Render }}, err = builder.meter.{{ $metric.Data.Instrument }}(
+        "{{ $name }}",
+        metric.WithDescription("{{ $metric.Description }}"),
+        metric.WithUnit("{{ $metric.Unit }}"),
+        {{- if eq $metric.Data.Type "Histogram" -}}
+        {{ if $metric.Data.Boundaries -}}metric.WithExplicitBucketBoundaries([]float64{ {{- range $metric.Data.Boundaries }} {{.}}, {{- end }} }...),{{- end }}
+        {{- end }}
+        {{ if $metric.Data.Async -}}
+        metric.With{{ casesTitle $metric.Data.BasicType }}Callback(func(_ context.Context, o metric.{{ casesTitle $metric.Data.BasicType }}Observer) error {
+            o.Observe(cb(), metric.WithAttributeSet(builder.attributeSet))
+            return nil
+        }),
+        {{- end }}
+    )
+    return err
+}
+
+    {{- else }}
     {{ if $metric.Data.Async -}}
 // With{{ $name.Render }}Callback sets callback for observable {{ $name.Render }} metric.
 func With{{ $name.Render }}Callback(cb func() {{ $metric.Data.BasicType }}) telemetryBuilderOption {
@@ -68,6 +90,7 @@ func With{{ $name.Render }}Callback(cb func() {{ $metric.Data.BasicType }}) tele
     }
 }
     {{- end }}
+    {{- end }}
 {{- end }}
 
 
@@ -86,6 +109,7 @@ func NewTelemetryBuilder(settings component.TelemetrySettings, options ...teleme
     }
 
     {{- range $name, $metric := .Telemetry.Metrics }}
+    {{- if not $metric.Optional }}
     builder.{{ $name.Render }}, err = builder.meter.{{ $metric.Data.Instrument }}(
         "{{ $name }}",
         metric.WithDescription("{{ $metric.Description }}"),
@@ -102,6 +126,7 @@ func NewTelemetryBuilder(settings component.TelemetrySettings, options ...teleme
     )
     errs = errors.Join(errs, err)
     {{- end }}
+    {{- end }}
     return &builder, errs
 }
 

cmd/otelcorecol/go.mod

@@ -64,14 +64,14 @@ require (
 	github.com/mitchellh/reflectwalk v1.0.2 // indirect
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
 	github.com/modern-go/reflect2 v1.0.2 // indirect
-	github.com/mostynb/go-grpc-compression v1.2.2 // indirect
+	github.com/mostynb/go-grpc-compression v1.2.3 // indirect
 	github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect
 	github.com/prometheus/client_golang v1.19.1 // indirect
 	github.com/prometheus/client_model v0.6.1 // indirect
 	github.com/prometheus/common v0.54.0 // indirect
 	github.com/prometheus/procfs v0.15.0 // indirect
 	github.com/rs/cors v1.10.1 // indirect
-	github.com/shirou/gopsutil/v3 v3.24.5 // indirect
+	github.com/shirou/gopsutil/v4 v4.24.5 // indirect
 	github.com/shoenig/go-m1cpu v0.1.6 // indirect
 	github.com/spf13/cobra v1.8.0 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
@@ -125,7 +125,7 @@ require (
 	google.golang.org/genproto/googleapis/api v0.0.0-20240520151616-dc85e6b867a5 // indirect
 	google.golang.org/genproto/googleapis/rpc v0.0.0-20240520151616-dc85e6b867a5 // indirect
 	google.golang.org/grpc v1.64.0 // indirect
-	google.golang.org/protobuf v1.34.1 // indirect
+	google.golang.org/protobuf v1.34.2 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 )