-
Notifications
You must be signed in to change notification settings - Fork 1.5k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Add telemetry for dropped data due to exporter sending queue overflow (…
…#3328) * Add telemetry for dropped data due to exporter sending queue overflow This change adds internal metrics for dropped spans, metric points and log records when exporter sending queue is full: - exporter/enqueue_failed_metric_points - exporter/enqueue_failed_spans - exporter/enqueue_failed_log_records * Make report*EnqueueFailure methods private By moving them to the package where they are being used. It requires some code duplication
- Loading branch information
Showing
13 changed files
with
298 additions
and
26 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,58 @@ | ||
| // Copyright The OpenTelemetry Authors | ||
| // | ||
| // Licensed under the Apache License, Version 2.0 (the "License"); | ||
| // you may not use this file except in compliance with the License. | ||
| // You may obtain a copy of the License at | ||
| // | ||
| // http://www.apache.org/licenses/LICENSE-2.0 | ||
| // | ||
| // Unless required by applicable law or agreed to in writing, software | ||
| // distributed under the License is distributed on an "AS IS" BASIS, | ||
| // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| // See the License for the specific language governing permissions and | ||
| // limitations under the License. | ||
|
|
||
| package exporterhelper | ||
|
|
||
| import ( | ||
| "context" | ||
|
|
||
| "go.opencensus.io/stats" | ||
| "go.opencensus.io/tag" | ||
|
|
||
| "go.opentelemetry.io/collector/internal/obsreportconfig/obsmetrics" | ||
| "go.opentelemetry.io/collector/obsreport" | ||
| ) | ||
|
|
||
| // TODO: Incorporate this functionality along with tests from obsreport_test.go | ||
| // into existing `obsreport` package once its functionally is not exposed | ||
| // as public API. For now this part is kept private. | ||
|
|
||
| // obsExporter is a helper to add observability to a component.Exporter. | ||
| type obsExporter struct { | ||
| *obsreport.Exporter | ||
| mutators []tag.Mutator | ||
| } | ||
|
|
||
| // newObsExporter creates a new observability exporter. | ||
| func newObsExporter(cfg obsreport.ExporterSettings) *obsExporter { | ||
| return &obsExporter{ | ||
| obsreport.NewExporter(cfg), | ||
| []tag.Mutator{tag.Upsert(obsmetrics.TagKeyExporter, cfg.ExporterID.String(), tag.WithTTL(tag.TTLNoPropagation))}, | ||
| } | ||
| } | ||
|
|
||
| // recordTracesEnqueueFailure records number of spans that failed to be added to the sending queue. | ||
| func (eor *obsExporter) recordTracesEnqueueFailure(ctx context.Context, numSpans int) { | ||
| _ = stats.RecordWithTags(ctx, eor.mutators, obsmetrics.ExporterFailedToEnqueueSpans.M(int64(numSpans))) | ||
| } | ||
|
|
||
| // recordMetricsEnqueueFailure records number of metric points that failed to be added to the sending queue. | ||
| func (eor *obsExporter) recordMetricsEnqueueFailure(ctx context.Context, numMetricPoints int) { | ||
| _ = stats.RecordWithTags(ctx, eor.mutators, obsmetrics.ExporterFailedToEnqueueMetricPoints.M(int64(numMetricPoints))) | ||
| } | ||
|
|
||
| // recordLogsEnqueueFailure records number of log records that failed to be added to the sending queue. | ||
| func (eor *obsExporter) recordLogsEnqueueFailure(ctx context.Context, numLogRecords int) { | ||
| _ = stats.RecordWithTags(ctx, eor.mutators, obsmetrics.ExporterFailedToEnqueueLogRecords.M(int64(numLogRecords))) | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,109 @@ | ||
| // Copyright The OpenTelemetry Authors | ||
| // | ||
| // Licensed under the Apache License, Version 2.0 (the "License"); | ||
| // you may not use this file except in compliance with the License. | ||
| // You may obtain a copy of the License at | ||
| // | ||
| // http://www.apache.org/licenses/LICENSE-2.0 | ||
| // | ||
| // Unless required by applicable law or agreed to in writing, software | ||
| // distributed under the License is distributed on an "AS IS" BASIS, | ||
| // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| // See the License for the specific language governing permissions and | ||
| // limitations under the License. | ||
|
|
||
| package exporterhelper | ||
|
|
||
| import ( | ||
| "context" | ||
| "reflect" | ||
| "sort" | ||
| "testing" | ||
|
|
||
| "github.com/stretchr/testify/require" | ||
| "go.opencensus.io/stats/view" | ||
| "go.opencensus.io/tag" | ||
|
|
||
| "go.opentelemetry.io/collector/config" | ||
| "go.opentelemetry.io/collector/config/configtelemetry" | ||
| "go.opentelemetry.io/collector/obsreport" | ||
| "go.opentelemetry.io/collector/obsreport/obsreporttest" | ||
| ) | ||
|
|
||
| func TestExportEnqueueFailure(t *testing.T) { | ||
| doneFn, err := obsreporttest.SetupRecordedMetricsTest() | ||
| require.NoError(t, err) | ||
| defer doneFn() | ||
|
|
||
| exporter := config.NewID("fakeExporter") | ||
|
|
||
| obsrep := newObsExporter(obsreport.ExporterSettings{Level: configtelemetry.LevelNormal, ExporterID: exporter}) | ||
|
|
||
| logRecords := 7 | ||
| obsrep.recordLogsEnqueueFailure(context.Background(), logRecords) | ||
| checkExporterEnqueueFailedLogsStats(t, exporter, int64(logRecords)) | ||
|
|
||
| spans := 12 | ||
| obsrep.recordTracesEnqueueFailure(context.Background(), spans) | ||
| checkExporterEnqueueFailedTracesStats(t, exporter, int64(spans)) | ||
|
|
||
| metricPoints := 21 | ||
| obsrep.recordMetricsEnqueueFailure(context.Background(), metricPoints) | ||
| checkExporterEnqueueFailedMetricsStats(t, exporter, int64(metricPoints)) | ||
| } | ||
|
|
||
| // checkExporterEnqueueFailedTracesStats checks that reported number of spans failed to enqueue match given values. | ||
| // When this function is called it is required to also call SetupRecordedMetricsTest as first thing. | ||
| func checkExporterEnqueueFailedTracesStats(t *testing.T, exporter config.ComponentID, spans int64) { | ||
| exporterTags := tagsForExporterView(exporter) | ||
| checkValueForView(t, exporterTags, spans, "exporter/enqueue_failed_spans") | ||
| } | ||
|
|
||
| // checkExporterEnqueueFailedMetricsStats checks that reported number of metric points failed to enqueue match given values. | ||
| // When this function is called it is required to also call SetupRecordedMetricsTest as first thing. | ||
| func checkExporterEnqueueFailedMetricsStats(t *testing.T, exporter config.ComponentID, metricPoints int64) { | ||
| exporterTags := tagsForExporterView(exporter) | ||
| checkValueForView(t, exporterTags, metricPoints, "exporter/enqueue_failed_metric_points") | ||
| } | ||
|
|
||
| // checkExporterEnqueueFailedLogsStats checks that reported number of log records failed to enqueue match given values. | ||
| // When this function is called it is required to also call SetupRecordedMetricsTest as first thing. | ||
| func checkExporterEnqueueFailedLogsStats(t *testing.T, exporter config.ComponentID, logRecords int64) { | ||
| exporterTags := tagsForExporterView(exporter) | ||
| checkValueForView(t, exporterTags, logRecords, "exporter/enqueue_failed_log_records") | ||
| } | ||
|
|
||
| // checkValueForView checks that for the current exported value in the view with the given name | ||
| // for {LegacyTagKeyReceiver: receiverName} is equal to "value". | ||
| func checkValueForView(t *testing.T, wantTags []tag.Tag, value int64, vName string) { | ||
| // Make sure the tags slice is sorted by tag keys. | ||
| sortTags(wantTags) | ||
|
|
||
| rows, err := view.RetrieveData(vName) | ||
| require.NoError(t, err) | ||
|
|
||
| for _, row := range rows { | ||
| // Make sure the tags slice is sorted by tag keys. | ||
| sortTags(row.Tags) | ||
| if reflect.DeepEqual(wantTags, row.Tags) { | ||
| sum := row.Data.(*view.SumData) | ||
| require.Equal(t, float64(value), sum.Value) | ||
| return | ||
| } | ||
| } | ||
|
|
||
| require.Failf(t, "could not find tags", "wantTags: %s in rows %v", wantTags, rows) | ||
| } | ||
|
|
||
| // tagsForExporterView returns the tags that are needed for the exporter views. | ||
| func tagsForExporterView(exporter config.ComponentID) []tag.Tag { | ||
| return []tag.Tag{ | ||
| {Key: exporterTag, Value: exporter.String()}, | ||
| } | ||
| } | ||
|
|
||
| func sortTags(tags []tag.Tag) { | ||
| sort.SliceStable(tags, func(i, j int) bool { | ||
| return tags[i].Key.Name() < tags[j].Key.Name() | ||
| }) | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.