From 45981b3f9140cddc5752f4e0a9dc1c1df67a4440 Mon Sep 17 00:00:00 2001 From: aishyandapalli Date: Tue, 31 Oct 2023 05:52:26 -0700 Subject: [PATCH] [Spanmetrics] Add events_total metric to get the measurement for list of configured event attributes for a span (#27811) **Description:** We have an events section for a span. The details for all the exceptions like exception.type and exception.message are recorded as Events for a span. Right now, we don't have a feature to add event attributes to span metrics. The idea of this PR is to develop a feature which adds a new metric `events_total` with a default set of dimensions like `service_name, span_name, span_kind, status_code`. We can configure to add additional set of dimensions like `exception.type` and `exception.message` which will be fetched from the Events section for a span **Link to tracking Issue:** [27451](https://github.com/open-telemetry/opentelemetry-collector-contrib/issues/27451) --------- Co-authored-by: Albert <26584478+albertteoh@users.noreply.github.com> --- .chloggen/events-metric-to-span-metrics.yaml | 27 +++++++++ connector/spanmetricsconnector/README.md | 10 +++- connector/spanmetricsconnector/config.go | 30 +++++++++- connector/spanmetricsconnector/config_test.go | 44 ++++++++++++++ connector/spanmetricsconnector/connector.go | 55 ++++++++++++++--- .../spanmetricsconnector/connector_test.go | 59 +++++++++++++++++++ 6 files changed, 214 insertions(+), 11 deletions(-) create mode 100644 .chloggen/events-metric-to-span-metrics.yaml diff --git a/.chloggen/events-metric-to-span-metrics.yaml b/.chloggen/events-metric-to-span-metrics.yaml new file mode 100644 index 000000000000..e1e12f564397 --- /dev/null +++ b/.chloggen/events-metric-to-span-metrics.yaml @@ -0,0 +1,27 @@ +# Use this changelog template to create an entry for release notes. + +# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix' +change_type: enhancement + +# The name of the component, or a single word describing the area of concern, (e.g. filelogreceiver) +component: spanmetricsconnector + +# A brief description of the change. Surround your text with quotes ("") if it needs to start with a backtick (`). +note: Add Events metric to span metrics connector that adds list of event attributes as dimensions + +# Mandatory: One or more tracking issues related to the change. You can use the PR number here if no issue exists. +issues: [27451] + +# (Optional) One or more lines of additional information to render under the primary note. +# These lines will be padded with 2 spaces and then inserted directly into the document. +# Use pipe (|) for multiline entries. +subtext: + +# If your change doesn't affect end users or the exported elements of any package, +# you should instead start your pull request title with [chore] or use the "Skip Changelog" label. +# Optional: The change log or logs in which this entry should be included. +# e.g. '[user]' or '[user, api]' +# Include 'user' if the change is relevant to end users. +# Include 'api' if there is a change to a library API. +# Default: '[user]' +change_logs: [] diff --git a/connector/spanmetricsconnector/README.md b/connector/spanmetricsconnector/README.md index 679522e67101..c81086f9c9b8 100644 --- a/connector/spanmetricsconnector/README.md +++ b/connector/spanmetricsconnector/README.md @@ -102,6 +102,9 @@ The following settings can be optionally configured: - `metrics_flush_interval` (default: `15s`): Defines the flush interval of the generated metrics. - `exemplars`: Use to configure how to attach exemplars to histograms - `enabled` (default: `false`): enabling will add spans as Exemplars. +- `events`: Use to configure the events metric. + - `enabled`: (default: `false`): enabling will add the events metric. + - `dimensions`: (mandatory if `enabled`) the list of the span's event attributes to add as dimensions to the events metric, which will be included _on top of_ the common and configured `dimensions` for span and resource attributes. ## Examples @@ -132,7 +135,12 @@ connectors: exclude_dimensions: ['status.code'] dimensions_cache_size: 1000 aggregation_temporality: "AGGREGATION_TEMPORALITY_CUMULATIVE" - metrics_flush_interval: 15s + metrics_flush_interval: 15s + events: + enabled: true + dimensions: + - name: exception.type + - name: exception.message service: pipelines: diff --git a/connector/spanmetricsconnector/config.go b/connector/spanmetricsconnector/config.go index 6eb5197b8a5e..26d36d785c60 100644 --- a/connector/spanmetricsconnector/config.go +++ b/connector/spanmetricsconnector/config.go @@ -58,6 +58,9 @@ type Config struct { // Exemplars defines the configuration for exemplars. Exemplars ExemplarsConfig `mapstructure:"exemplars"` + + // Events defines the configuration for events section of spans. + Events EventsConfig `mapstructure:"events"` } type HistogramConfig struct { @@ -80,13 +83,22 @@ type ExplicitHistogramConfig struct { Buckets []time.Duration `mapstructure:"buckets"` } +type EventsConfig struct { + // Enabled is a flag to enable events. + Enabled bool `mapstructure:"enabled"` + // Dimensions defines the list of dimensions to add to the events metric. + Dimensions []Dimension `mapstructure:"dimensions"` +} + var _ component.ConfigValidator = (*Config)(nil) // Validate checks if the processor configuration is valid func (c Config) Validate() error { - err := validateDimensions(c.Dimensions) - if err != nil { - return err + if err := validateDimensions(c.Dimensions); err != nil { + return fmt.Errorf("failed validating dimensions: %w", err) + } + if err := validateEventDimensions(c.Events.Enabled, c.Events.Dimensions); err != nil { + return fmt.Errorf("failed validating event dimensions: %w", err) } if c.DimensionsCacheSize <= 0 { @@ -99,6 +111,7 @@ func (c Config) Validate() error { if c.Histogram.Explicit != nil && c.Histogram.Exponential != nil { return errors.New("use either `explicit` or `exponential` buckets histogram") } + return nil } @@ -127,3 +140,14 @@ func validateDimensions(dimensions []Dimension) error { return nil } + +// validateEventDimensions checks for empty and duplicates for the dimensions configured. +func validateEventDimensions(enabled bool, dimensions []Dimension) error { + if !enabled { + return nil + } + if len(dimensions) == 0 { + return fmt.Errorf("no dimensions configured for events") + } + return validateDimensions(dimensions) +} diff --git a/connector/spanmetricsconnector/config_test.go b/connector/spanmetricsconnector/config_test.go index 31505a2de16a..2530c481d27b 100644 --- a/connector/spanmetricsconnector/config_test.go +++ b/connector/spanmetricsconnector/config_test.go @@ -171,3 +171,47 @@ func TestValidateDimensions(t *testing.T) { }) } } + +func TestValidateEventDimensions(t *testing.T) { + for _, tc := range []struct { + enabled bool + name string + dimensions []Dimension + expectedErr string + }{ + { + enabled: false, + name: "disabled - no additional dimensions", + dimensions: []Dimension{}, + }, + { + enabled: true, + name: "enabled - no additional dimensions", + dimensions: []Dimension{}, + expectedErr: "no dimensions configured for events", + }, + { + enabled: true, + name: "enabled - no duplicate dimensions", + dimensions: []Dimension{{Name: "exception_type"}}, + }, + { + enabled: true, + name: "enabled - duplicate dimensions", + dimensions: []Dimension{ + {Name: "exception_type"}, + {Name: "exception_type"}, + }, + expectedErr: "duplicate dimension name exception_type", + }, + } { + t.Run(tc.name, func(t *testing.T) { + err := validateEventDimensions(tc.enabled, tc.dimensions) + if tc.expectedErr != "" { + assert.EqualError(t, err, tc.expectedErr) + } else { + assert.NoError(t, err) + } + }) + } +} diff --git a/connector/spanmetricsconnector/connector.go b/connector/spanmetricsconnector/connector.go index 1661f03823d8..9e0189f1f921 100644 --- a/connector/spanmetricsconnector/connector.go +++ b/connector/spanmetricsconnector/connector.go @@ -36,6 +36,7 @@ const ( metricNameDuration = "duration" metricNameCalls = "calls" + metricNameEvents = "events" defaultUnit = metrics.Milliseconds ) @@ -66,11 +67,17 @@ type connectorImp struct { started bool shutdownOnce sync.Once + + // Event dimensions to add to the events metric. + eDimensions []dimension + + events EventsConfig } type resourceMetrics struct { histograms metrics.HistogramMetrics sums metrics.SumMetrics + events metrics.SumMetrics attributes pcommon.Map } @@ -113,6 +120,8 @@ func newConnector(logger *zap.Logger, config component.Config, ticker *clock.Tic metricKeyToDimensions: metricKeyToDimensionsCache, ticker: ticker, done: make(chan struct{}), + eDimensions: newDimensions(cfg.Events.Dimensions), + events: cfg.Events, }, nil } @@ -245,6 +254,13 @@ func (p *connectorImp) buildMetrics() pmetric.Metrics { metric.SetUnit(p.config.Histogram.Unit.String()) histograms.BuildMetrics(metric, p.startTimestamp, p.config.GetAggregationTemporality()) } + + events := rawMetrics.events + if p.events.Enabled { + metric = sm.Metrics().AppendEmpty() + metric.SetName(buildMetricName(p.config.Namespace, metricNameEvents)) + events.BuildMetrics(metric, p.startTimestamp, p.config.GetAggregationTemporality()) + } } return m @@ -288,6 +304,7 @@ func (p *connectorImp) aggregateMetrics(traces ptrace.Traces) { rm := p.getOrCreateResourceMetrics(resourceAttr) sums := rm.sums histograms := rm.histograms + events := rm.events unitDivider := unitDivider(p.config.Histogram.Unit) serviceName := serviceAttr.Str() @@ -308,7 +325,7 @@ func (p *connectorImp) aggregateMetrics(traces ptrace.Traces) { attributes, ok := p.metricKeyToDimensions.Get(key) if !ok { - attributes = p.buildAttributes(serviceName, span, resourceAttr) + attributes = p.buildAttributes(serviceName, span, resourceAttr, p.dimensions) p.metricKeyToDimensions.Add(key, attributes) } if !p.config.Histogram.Disable { @@ -321,6 +338,29 @@ func (p *connectorImp) aggregateMetrics(traces ptrace.Traces) { // aggregate sums metrics s := sums.GetOrCreate(key, attributes) s.Add(1) + + // aggregate events metrics + if p.events.Enabled { + for l := 0; l < span.Events().Len(); l++ { + event := span.Events().At(l) + eDimensions := p.dimensions + eDimensions = append(eDimensions, p.eDimensions...) + + rscAndEventAttrs := pcommon.NewMap() + rscAndEventAttrs.EnsureCapacity(resourceAttr.Len() + event.Attributes().Len()) + resourceAttr.CopyTo(rscAndEventAttrs) + event.Attributes().CopyTo(rscAndEventAttrs) + + eKey := p.buildKey(serviceName, span, eDimensions, rscAndEventAttrs) + eAttributes, ok := p.metricKeyToDimensions.Get(eKey) + if !ok { + eAttributes = p.buildAttributes(serviceName, span, rscAndEventAttrs, eDimensions) + p.metricKeyToDimensions.Add(eKey, eAttributes) + } + e := events.GetOrCreate(eKey, eAttributes) + e.Add(1) + } + } } } } @@ -346,6 +386,7 @@ func (p *connectorImp) getOrCreateResourceMetrics(attr pcommon.Map) *resourceMet v = &resourceMetrics{ histograms: initHistogramMetrics(p.config), sums: metrics.NewSumMetrics(), + events: metrics.NewSumMetrics(), attributes: attr, } p.resourceMetrics[key] = v @@ -363,9 +404,9 @@ func contains(elements []string, value string) bool { return false } -func (p *connectorImp) buildAttributes(serviceName string, span ptrace.Span, resourceAttrs pcommon.Map) pcommon.Map { +func (p *connectorImp) buildAttributes(serviceName string, span ptrace.Span, resourceAttrs pcommon.Map, dimensions []dimension) pcommon.Map { attr := pcommon.NewMap() - attr.EnsureCapacity(4 + len(p.dimensions)) + attr.EnsureCapacity(4 + len(dimensions)) if !contains(p.config.ExcludeDimensions, serviceNameKey) { attr.PutStr(serviceNameKey, serviceName) } @@ -378,7 +419,7 @@ func (p *connectorImp) buildAttributes(serviceName string, span ptrace.Span, res if !contains(p.config.ExcludeDimensions, statusCodeKey) { attr.PutStr(statusCodeKey, traceutil.StatusCodeStr(span.Status().Code())) } - for _, d := range p.dimensions { + for _, d := range dimensions { if v, ok := getDimensionValue(d, span.Attributes(), resourceAttrs); ok { v.CopyTo(attr.PutEmpty(d.name)) } @@ -395,10 +436,10 @@ func concatDimensionValue(dest *bytes.Buffer, value string, prefixSep bool) { // buildKey builds the metric key from the service name and span metadata such as name, kind, status_code and // will attempt to add any additional dimensions the user has configured that match the span's attributes -// or resource attributes. If the dimension exists in both, the span's attributes, being the most specific, takes precedence. +// or resource/event attributes. If the dimension exists in both, the span's attributes, being the most specific, takes precedence. // // The metric key is a simple concatenation of dimension values, delimited by a null character. -func (p *connectorImp) buildKey(serviceName string, span ptrace.Span, optionalDims []dimension, resourceAttrs pcommon.Map) metrics.Key { +func (p *connectorImp) buildKey(serviceName string, span ptrace.Span, optionalDims []dimension, resourceOrEventAttrs pcommon.Map) metrics.Key { p.keyBuf.Reset() if !contains(p.config.ExcludeDimensions, serviceNameKey) { concatDimensionValue(p.keyBuf, serviceName, false) @@ -414,7 +455,7 @@ func (p *connectorImp) buildKey(serviceName string, span ptrace.Span, optionalDi } for _, d := range optionalDims { - if v, ok := getDimensionValue(d, span.Attributes(), resourceAttrs); ok { + if v, ok := getDimensionValue(d, span.Attributes(), resourceOrEventAttrs); ok { concatDimensionValue(p.keyBuf, v.AsString(), true) } } diff --git a/connector/spanmetricsconnector/connector_test.go b/connector/spanmetricsconnector/connector_test.go index 0ae24ac4b7c5..2e3e3c34c7cb 100644 --- a/connector/spanmetricsconnector/connector_test.go +++ b/connector/spanmetricsconnector/connector_test.go @@ -42,6 +42,7 @@ const ( notInSpanAttrName0 = "shouldBeInMetric" notInSpanAttrName1 = "shouldNotBeInMetric" regionResourceAttrName = "region" + exceptionTypeAttrName = "exception.type" DimensionsCacheSize = 2 sampleRegion = "us-east-1" @@ -394,6 +395,10 @@ func initSpan(span span, s ptrace.Span) { s.Attributes().PutEmptySlice(arrayAttrName) s.SetTraceID(pcommon.TraceID(span.traceID)) s.SetSpanID(pcommon.SpanID(span.spanID)) + + e := s.Events().AppendEmpty() + e.SetName("exception") + e.Attributes().PutStr(exceptionTypeAttrName, "NullPointerException") } func disabledExemplarsConfig() ExemplarsConfig { @@ -1259,3 +1264,57 @@ func TestConnector_initHistogramMetrics(t *testing.T) { }) } } + +func TestSpanMetrics_Events(t *testing.T) { + tests := []struct { + name string + eventsConfig EventsConfig + shouldEventsMetricExist bool + }{ + { + name: "events disabled", + eventsConfig: EventsConfig{Enabled: false, Dimensions: []Dimension{{Name: "exception.type", Default: stringp("NullPointerException")}}}, + shouldEventsMetricExist: false, + }, + { + name: "events enabled", + eventsConfig: EventsConfig{Enabled: true, Dimensions: []Dimension{{Name: "exception.type", Default: stringp("NullPointerException")}}}, + shouldEventsMetricExist: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + factory := NewFactory() + cfg := factory.CreateDefaultConfig().(*Config) + cfg.Events = tt.eventsConfig + c, err := newConnector(zaptest.NewLogger(t), cfg, nil) + require.NoError(t, err) + err = c.ConsumeTraces(context.Background(), buildSampleTrace()) + require.NoError(t, err) + metrics := c.buildMetrics() + for i := 0; i < metrics.ResourceMetrics().Len(); i++ { + rm := metrics.ResourceMetrics().At(i) + ism := rm.ScopeMetrics() + for ilmC := 0; ilmC < ism.Len(); ilmC++ { + m := ism.At(ilmC).Metrics() + if !tt.shouldEventsMetricExist { + assert.Equal(t, 2, m.Len()) + continue + } + assert.Equal(t, 3, m.Len()) + for mC := 0; mC < m.Len(); mC++ { + metric := m.At(mC) + if metric.Name() != "events" { + continue + } + assert.Equal(t, pmetric.MetricTypeSum, metric.Type()) + for idp := 0; idp < metric.Sum().DataPoints().Len(); idp++ { + attrs := metric.Sum().DataPoints().At(idp).Attributes() + assert.Contains(t, attrs.AsRaw(), exceptionTypeAttrName) + } + } + } + } + }) + } +}