From ee3f4bc5c8195c554e2c31f009ae07469bd8886d Mon Sep 17 00:00:00 2001 From: Dmitry Anoshin Date: Thu, 14 Mar 2024 16:20:21 -0700 Subject: [PATCH] Distributed internal metrics across different levels The internal metrics levels are updated along with reported metrics: - The default level is changed from `basic` to `normal`, which can be overridden with `service::telmetry::metrics::level` configuration. - Batch processor metrics are updated to be reported starting from `normal` level: - `processor_batch_batch_send_size` - `processor_batch_metadata_cardinality` - `processor_batch_timeout_trigger_send` - `processor_batch_size_trigger_send` - GRPC/HTTP server and client metrics are updated to be reported starting from `detailed` level: - http.client.* metrics - http.server.* metrics - rpc.server.* metrics - rpc.client.* metrics --- .chloggen/update-internal-metric-levels.yaml | 37 ++++++++++++++++++++ config/configgrpc/configgrpc.go | 9 +++-- config/configgrpc/go.mod | 2 +- config/confighttp/confighttp.go | 32 ++++++++++------- otelcol/unmarshaler.go | 2 +- processor/batchprocessor/metrics.go | 14 ++++++-- 6 files changed, 77 insertions(+), 19 deletions(-) create mode 100755 .chloggen/update-internal-metric-levels.yaml diff --git a/.chloggen/update-internal-metric-levels.yaml b/.chloggen/update-internal-metric-levels.yaml new file mode 100755 index 00000000000..6870096bf8f --- /dev/null +++ b/.chloggen/update-internal-metric-levels.yaml @@ -0,0 +1,37 @@ +# Use this changelog template to create an entry for release notes. + +# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix' +change_type: breaking + +# The name of the component, or a single word describing the area of concern, (e.g. otlpreceiver) +component: telemetry + +# A brief description of the change. Surround your text with quotes ("") if it needs to start with a backtick (`). +note: Distributed internal metrics across different levels. + +# One or more tracking issues or pull requests related to the change +issues: [7890] + +# (Optional) One or more lines of additional information to render under the primary note. +# These lines will be padded with 2 spaces and then inserted directly into the document. +# Use pipe (|) for multiline entries. +subtext: | + The internal metrics levels are updated along with reported metrics: + - The default level is changed from `basic` to `normal`, which can be overridden with `service::telmetry::metrics::level` configuration. + - Batch processor metrics are updated to be reported starting from `normal` level: + - `processor_batch_batch_send_size` + - `processor_batch_metadata_cardinality` + - `processor_batch_timeout_trigger_send` + - `processor_batch_size_trigger_send` + - GRPC/HTTP server and client metrics are updated to be reported starting from `detailed` level: + - http.client.* metrics + - http.server.* metrics + - rpc.server.* metrics + - rpc.client.* metrics + +# Optional: The change log or logs in which this entry should be included. +# e.g. '[user]' or '[user, api]' +# Include 'user' if the change is relevant to end users. +# Include 'api' if there is a change to a library API. +# Default: '[user]' +change_logs: [user] \ No newline at end of file diff --git a/config/configgrpc/configgrpc.go b/config/configgrpc/configgrpc.go index 71d108b6542..62375ff2d8f 100644 --- a/config/configgrpc/configgrpc.go +++ b/config/configgrpc/configgrpc.go @@ -30,6 +30,7 @@ import ( "go.opentelemetry.io/collector/config/configcompression" "go.opentelemetry.io/collector/config/confignet" "go.opentelemetry.io/collector/config/configopaque" + "go.opentelemetry.io/collector/config/configtelemetry" "go.opentelemetry.io/collector/config/configtls" "go.opentelemetry.io/collector/config/internal" "go.opentelemetry.io/collector/extension/auth" @@ -253,9 +254,11 @@ func (gcs *ClientConfig) toDialOptions(host component.Host, settings component.T otelOpts := []otelgrpc.Option{ otelgrpc.WithTracerProvider(settings.TracerProvider), - otelgrpc.WithMeterProvider(settings.MeterProvider), otelgrpc.WithPropagators(otel.GetTextMapPropagator()), } + if settings.MetricsLevel >= configtelemetry.LevelDetailed { + otelOpts = append(otelOpts, otelgrpc.WithMeterProvider(settings.MeterProvider)) + } // Enable OpenTelemetry observability plugin. opts = append(opts, grpc.WithStatsHandler(otelgrpc.NewClientHandler(otelOpts...))) @@ -356,9 +359,11 @@ func (gss *ServerConfig) toServerOption(host component.Host, settings component. otelOpts := []otelgrpc.Option{ otelgrpc.WithTracerProvider(settings.TracerProvider), - otelgrpc.WithMeterProvider(settings.MeterProvider), otelgrpc.WithPropagators(otel.GetTextMapPropagator()), } + if settings.MetricsLevel >= configtelemetry.LevelDetailed { + otelOpts = append(otelOpts, otelgrpc.WithMeterProvider(settings.MeterProvider)) + } // Enable OpenTelemetry observability plugin. diff --git a/config/configgrpc/go.mod b/config/configgrpc/go.mod index 59ba0814322..8af35bf3197 100644 --- a/config/configgrpc/go.mod +++ b/config/configgrpc/go.mod @@ -11,6 +11,7 @@ require ( go.opentelemetry.io/collector/config/configcompression v1.5.0 go.opentelemetry.io/collector/config/confignet v0.98.0 go.opentelemetry.io/collector/config/configopaque v1.5.0 + go.opentelemetry.io/collector/config/configtelemetry v0.98.0 go.opentelemetry.io/collector/config/configtls v0.98.0 go.opentelemetry.io/collector/config/internal v0.98.0 go.opentelemetry.io/collector/extension/auth v0.98.0 @@ -48,7 +49,6 @@ require ( github.com/prometheus/client_model v0.6.1 // indirect github.com/prometheus/common v0.52.3 // indirect github.com/prometheus/procfs v0.12.0 // indirect - go.opentelemetry.io/collector/config/configtelemetry v0.98.0 // indirect go.opentelemetry.io/collector/confmap v0.98.0 // indirect go.opentelemetry.io/collector/extension v0.98.0 // indirect go.opentelemetry.io/collector/featuregate v1.5.0 // indirect diff --git a/config/confighttp/confighttp.go b/config/confighttp/confighttp.go index 6b408472a43..303075571db 100644 --- a/config/confighttp/confighttp.go +++ b/config/confighttp/confighttp.go @@ -23,6 +23,7 @@ import ( "go.opentelemetry.io/collector/config/configauth" "go.opentelemetry.io/collector/config/configcompression" "go.opentelemetry.io/collector/config/configopaque" + "go.opentelemetry.io/collector/config/configtelemetry" "go.opentelemetry.io/collector/config/configtls" "go.opentelemetry.io/collector/config/internal" "go.opentelemetry.io/collector/extension/auth" @@ -206,14 +207,17 @@ func (hcs *ClientConfig) ToClient(ctx context.Context, host component.Host, sett } } + otelOpts := []otelhttp.Option{ + otelhttp.WithTracerProvider(settings.TracerProvider), + otelhttp.WithPropagators(otel.GetTextMapPropagator()), + } + if settings.MetricsLevel >= configtelemetry.LevelDetailed { + otelOpts = append(otelOpts, otelhttp.WithMeterProvider(settings.MeterProvider)) + } + // wrapping http transport with otelhttp transport to enable otel instrumentation if settings.TracerProvider != nil && settings.MeterProvider != nil { - clientTransport = otelhttp.NewTransport( - clientTransport, - otelhttp.WithTracerProvider(settings.TracerProvider), - otelhttp.WithMeterProvider(settings.MeterProvider), - otelhttp.WithPropagators(otel.GetTextMapPropagator()), - ) + clientTransport = otelhttp.NewTransport(clientTransport, otelOpts...) } if hcs.CustomRoundTripper != nil { @@ -383,18 +387,20 @@ func (hss *ServerConfig) ToServer(_ context.Context, host component.Host, settin handler = responseHeadersHandler(handler, hss.ResponseHeaders) } - // Enable OpenTelemetry observability plugin. - // TODO: Consider to use component ID string as prefix for all the operations. - handler = otelhttp.NewHandler( - handler, - "", + otelOpts := []otelhttp.Option{ otelhttp.WithTracerProvider(settings.TracerProvider), - otelhttp.WithMeterProvider(settings.MeterProvider), otelhttp.WithPropagators(otel.GetTextMapPropagator()), otelhttp.WithSpanNameFormatter(func(_ string, r *http.Request) string { return r.URL.Path }), - ) + } + if settings.MetricsLevel >= configtelemetry.LevelDetailed { + otelOpts = append(otelOpts, otelhttp.WithMeterProvider(settings.MeterProvider)) + } + + // Enable OpenTelemetry observability plugin. + // TODO: Consider to use component ID string as prefix for all the operations. + handler = otelhttp.NewHandler(handler, "", otelOpts...) // wrap the current handler in an interceptor that will add client.Info to the request's context handler = &clientInfoHandler{ diff --git a/otelcol/unmarshaler.go b/otelcol/unmarshaler.go index 643e396db42..a522d9fa60d 100644 --- a/otelcol/unmarshaler.go +++ b/otelcol/unmarshaler.go @@ -59,7 +59,7 @@ func unmarshal(v *confmap.Conf, factories Factories) (*configSettings, error) { InitialFields: map[string]any(nil), }, Metrics: telemetry.MetricsConfig{ - Level: configtelemetry.LevelBasic, + Level: configtelemetry.LevelNormal, Address: ":8888", }, }, diff --git a/processor/batchprocessor/metrics.go b/processor/batchprocessor/metrics.go index 56ee5f3025b..95fe7f28e16 100644 --- a/processor/batchprocessor/metrics.go +++ b/processor/batchprocessor/metrics.go @@ -8,6 +8,7 @@ import ( "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/metric" + noopmetric "go.opentelemetry.io/otel/metric/noop" "go.uber.org/multierr" "go.opentelemetry.io/collector/component" @@ -56,8 +57,17 @@ func newBatchProcessorTelemetry(set processor.CreateSettings, currentMetadataCar } func (bpt *batchProcessorTelemetry) createOtelMetrics(set component.TelemetrySettings, currentMetadataCardinality func() int) error { - var errors, err error - meter := metadata.Meter(set) + var ( + errors, err error + meter metric.Meter + ) + + // BatchProcessor are emitted starting from Normal level only. + if bpt.level >= configtelemetry.LevelNormal { + meter = metadata.Meter(set) + } else { + meter = noopmetric.Meter{} + } bpt.batchSizeTriggerSend, err = meter.Int64Counter( processorhelper.BuildCustomMetricName(typeStr, "batch_size_trigger_send"),