From 7a1130eacb94b7d007b00f182e929f467c388c36 Mon Sep 17 00:00:00 2001 From: Bogdan Drutu Date: Mon, 31 Aug 2020 14:52:57 -0700 Subject: [PATCH] Change OTLP exporter/receiver and File exporter to use the new opentelemetry proto v0.5.0 Signed-off-by: Bogdan Drutu --- CHANGELOG.md | 6 ++ exporter/fileexporter/file_exporter.go | 4 +- exporter/fileexporter/file_exporter_test.go | 7 +- exporter/otlpexporter/otlp.go | 6 +- exporter/otlpexporter/otlp_test.go | 12 +-- .../metrics/v1/metrics_service.pb.go | 2 +- receiver/otlpreceiver/metrics/otlp.go | 8 +- receiver/otlpreceiver/metrics/otlp_test.go | 84 ++++++++++--------- 8 files changed, 67 insertions(+), 62 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 10819367045..d35bab3ecc8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,12 @@ ## Unreleased +## 🛑 Breaking changes 🛑 + +- **Update OTLP to v0.5.0, incompatible metrics protocol.** +- Remove support for propagating summary metrics in OtelCollector. + - This is a temporary change, and will affect mostly OpenCensus users who use metrics. + ## v0.9.0 Beta ## 🛑 Breaking changes 🛑 diff --git a/exporter/fileexporter/file_exporter.go b/exporter/fileexporter/file_exporter.go index 98e788b9e3a..e8d57028e51 100644 --- a/exporter/fileexporter/file_exporter.go +++ b/exporter/fileexporter/file_exporter.go @@ -25,10 +25,10 @@ import ( "go.opentelemetry.io/collector/component" "go.opentelemetry.io/collector/consumer/pdata" "go.opentelemetry.io/collector/consumer/pdatautil" + "go.opentelemetry.io/collector/internal/data" otlplogs "go.opentelemetry.io/collector/internal/data/opentelemetry-proto-gen/collector/logs/v1" otlpmetrics "go.opentelemetry.io/collector/internal/data/opentelemetry-proto-gen/collector/metrics/v1" otlptrace "go.opentelemetry.io/collector/internal/data/opentelemetry-proto-gen/collector/trace/v1" - "go.opentelemetry.io/collector/internal/dataold" ) // Marshaler configuration used for marhsaling Protobuf to JSON. Use default config. @@ -50,7 +50,7 @@ func (e *fileExporter) ConsumeTraces(_ context.Context, td pdata.Traces) error { func (e *fileExporter) ConsumeMetrics(_ context.Context, md pdata.Metrics) error { request := otlpmetrics.ExportMetricsServiceRequest{ - ResourceMetrics: dataold.MetricDataToOtlp(pdatautil.MetricsToOldInternalMetrics(md)), + ResourceMetrics: data.MetricDataToOtlp(pdatautil.MetricsToInternalMetrics(md)), } return exportMessageAsLine(e, &request) } diff --git a/exporter/fileexporter/file_exporter_test.go b/exporter/fileexporter/file_exporter_test.go index 0ad9fd102e8..400ec0c459e 100644 --- a/exporter/fileexporter/file_exporter_test.go +++ b/exporter/fileexporter/file_exporter_test.go @@ -24,6 +24,7 @@ import ( "go.opentelemetry.io/collector/consumer/pdata" "go.opentelemetry.io/collector/consumer/pdatautil" + "go.opentelemetry.io/collector/internal/data" collectorlogs "go.opentelemetry.io/collector/internal/data/opentelemetry-proto-gen/collector/logs/v1" collectormetrics "go.opentelemetry.io/collector/internal/data/opentelemetry-proto-gen/collector/metrics/v1" collectortrace "go.opentelemetry.io/collector/internal/data/opentelemetry-proto-gen/collector/trace/v1" @@ -31,8 +32,6 @@ import ( logspb "go.opentelemetry.io/collector/internal/data/opentelemetry-proto-gen/logs/v1" otresourcepb "go.opentelemetry.io/collector/internal/data/opentelemetry-proto-gen/resource/v1" "go.opentelemetry.io/collector/internal/data/testdata" - "go.opentelemetry.io/collector/internal/dataold" - "go.opentelemetry.io/collector/internal/dataold/testdataold" "go.opentelemetry.io/collector/testutil" ) @@ -58,7 +57,7 @@ func TestFileMetricsExporterNoErrors(t *testing.T) { lme := &fileExporter{file: mf} require.NotNil(t, lme) - md := pdatautil.MetricsFromOldInternalMetrics(testdataold.GenerateMetricDataTwoMetrics()) + md := pdatautil.MetricsFromInternalMetrics(testdata.GenerateMetricsTwoMetrics()) assert.NoError(t, lme.ConsumeMetrics(context.Background(), md)) assert.NoError(t, lme.Shutdown(context.Background())) @@ -66,7 +65,7 @@ func TestFileMetricsExporterNoErrors(t *testing.T) { var j collectormetrics.ExportMetricsServiceRequest assert.NoError(t, unmarshaler.Unmarshal(mf, &j)) - assert.EqualValues(t, dataold.MetricDataToOtlp(pdatautil.MetricsToOldInternalMetrics(md)), j.ResourceMetrics) + assert.EqualValues(t, data.MetricDataToOtlp(pdatautil.MetricsToInternalMetrics(md)), j.ResourceMetrics) } func TestFileLogsExporterNoErrors(t *testing.T) { diff --git a/exporter/otlpexporter/otlp.go b/exporter/otlpexporter/otlp.go index ae3e705d782..cfa3bc805b4 100644 --- a/exporter/otlpexporter/otlp.go +++ b/exporter/otlpexporter/otlp.go @@ -31,10 +31,10 @@ import ( "go.opentelemetry.io/collector/consumer/pdata" "go.opentelemetry.io/collector/consumer/pdatautil" "go.opentelemetry.io/collector/exporter/exporterhelper" + "go.opentelemetry.io/collector/internal/data" otlplogs "go.opentelemetry.io/collector/internal/data/opentelemetry-proto-gen/collector/logs/v1" otlpmetrics "go.opentelemetry.io/collector/internal/data/opentelemetry-proto-gen/collector/metrics/v1" otlptrace "go.opentelemetry.io/collector/internal/data/opentelemetry-proto-gen/collector/trace/v1" - "go.opentelemetry.io/collector/internal/dataold" ) type exporterImp struct { @@ -90,9 +90,9 @@ func (e *exporterImp) pushTraceData(ctx context.Context, td pdata.Traces) (int, } func (e *exporterImp) pushMetricsData(ctx context.Context, md pdata.Metrics) (int, error) { - imd := pdatautil.MetricsToOldInternalMetrics(md) + imd := pdatautil.MetricsToInternalMetrics(md) request := &otlpmetrics.ExportMetricsServiceRequest{ - ResourceMetrics: dataold.MetricDataToOtlp(imd), + ResourceMetrics: data.MetricDataToOtlp(imd), } err := e.w.exportMetrics(ctx, request) diff --git a/exporter/otlpexporter/otlp_test.go b/exporter/otlpexporter/otlp_test.go index 9856dd817ed..12b8cfc1c31 100644 --- a/exporter/otlpexporter/otlp_test.go +++ b/exporter/otlpexporter/otlp_test.go @@ -33,6 +33,7 @@ import ( "go.opentelemetry.io/collector/config/configtls" "go.opentelemetry.io/collector/consumer/pdata" "go.opentelemetry.io/collector/consumer/pdatautil" + "go.opentelemetry.io/collector/internal/data" otlplogs "go.opentelemetry.io/collector/internal/data/opentelemetry-proto-gen/collector/logs/v1" otlpmetrics "go.opentelemetry.io/collector/internal/data/opentelemetry-proto-gen/collector/metrics/v1" otlptraces "go.opentelemetry.io/collector/internal/data/opentelemetry-proto-gen/collector/trace/v1" @@ -135,14 +136,7 @@ func (r *mockMetricsReceiver) Export( req *otlpmetrics.ExportMetricsServiceRequest, ) (*otlpmetrics.ExportMetricsServiceResponse, error) { atomic.AddInt32(&r.requestCount, 1) - recordCount := 0 - for _, rs := range req.ResourceMetrics { - for _, il := range rs.InstrumentationLibraryMetrics { - for _, m := range il.Metrics { - recordCount += len(m.DoubleDataPoints) + len(m.Int64DataPoints) + len(m.HistogramDataPoints) + len(m.SummaryDataPoints) - } - } - } + _, recordCount := data.MetricDataFromOtlp(req.ResourceMetrics).MetricAndDataPointCount() atomic.AddInt32(&r.totalItems, int32(recordCount)) r.lastRequest = req r.metadata, _ = metadata.FromIncomingContext(ctx) @@ -288,7 +282,7 @@ func TestSendMetrics(t *testing.T) { md = pdatautil.MetricsFromOldInternalMetrics(testdataold.GenerateMetricDataTwoMetrics()) expectedOTLPReq := &otlpmetrics.ExportMetricsServiceRequest{ - ResourceMetrics: testdataold.GenerateMetricOtlpTwoMetrics(), + ResourceMetrics: testdata.GenerateMetricsOtlpTwoMetrics(), } err = exp.ConsumeMetrics(context.Background(), md) diff --git a/internal/data/opentelemetry-proto-gen/collector/metrics/v1/metrics_service.pb.go b/internal/data/opentelemetry-proto-gen/collector/metrics/v1/metrics_service.pb.go index 3cf40ed8909..95ac23f43a7 100644 --- a/internal/data/opentelemetry-proto-gen/collector/metrics/v1/metrics_service.pb.go +++ b/internal/data/opentelemetry-proto-gen/collector/metrics/v1/metrics_service.pb.go @@ -15,7 +15,7 @@ import ( codes "google.golang.org/grpc/codes" status "google.golang.org/grpc/status" - v1 "go.opentelemetry.io/collector/internal/data/opentelemetry-proto-gen/metrics/v1old" + v1 "go.opentelemetry.io/collector/internal/data/opentelemetry-proto-gen/metrics/v1" ) // Reference imports to suppress errors if they are not otherwise used. diff --git a/receiver/otlpreceiver/metrics/otlp.go b/receiver/otlpreceiver/metrics/otlp.go index cb7dd008ef6..545a89e62dc 100644 --- a/receiver/otlpreceiver/metrics/otlp.go +++ b/receiver/otlpreceiver/metrics/otlp.go @@ -20,8 +20,8 @@ import ( "go.opentelemetry.io/collector/client" "go.opentelemetry.io/collector/consumer" "go.opentelemetry.io/collector/consumer/pdatautil" + "go.opentelemetry.io/collector/internal/data" collectormetrics "go.opentelemetry.io/collector/internal/data/opentelemetry-proto-gen/collector/metrics/v1" - "go.opentelemetry.io/collector/internal/dataold" "go.opentelemetry.io/collector/obsreport" ) @@ -52,7 +52,7 @@ const ( func (r *Receiver) Export(ctx context.Context, req *collectormetrics.ExportMetricsServiceRequest) (*collectormetrics.ExportMetricsServiceResponse, error) { receiverCtx := obsreport.ReceiverContext(ctx, r.instanceName, receiverTransport, receiverTagValue) - md := dataold.MetricDataFromOtlp(req.ResourceMetrics) + md := data.MetricDataFromOtlp(req.ResourceMetrics) err := r.sendToNextConsumer(receiverCtx, md) if err != nil { @@ -62,7 +62,7 @@ func (r *Receiver) Export(ctx context.Context, req *collectormetrics.ExportMetri return &collectormetrics.ExportMetricsServiceResponse{}, nil } -func (r *Receiver) sendToNextConsumer(ctx context.Context, md dataold.MetricData) error { +func (r *Receiver) sendToNextConsumer(ctx context.Context, md data.MetricData) error { metricCount, dataPointCount := md.MetricAndDataPointCount() if metricCount == 0 { return nil @@ -73,7 +73,7 @@ func (r *Receiver) sendToNextConsumer(ctx context.Context, md dataold.MetricData } ctx = obsreport.StartMetricsReceiveOp(ctx, r.instanceName, receiverTransport) - err := r.nextConsumer.ConsumeMetrics(ctx, pdatautil.MetricsFromOldInternalMetrics(md)) + err := r.nextConsumer.ConsumeMetrics(ctx, pdatautil.MetricsFromInternalMetrics(md)) obsreport.EndMetricsReceiveOp(ctx, dataFormatProtobuf, dataPointCount, metricCount, err) return err diff --git a/receiver/otlpreceiver/metrics/otlp_test.go b/receiver/otlpreceiver/metrics/otlp_test.go index 6919cb5c7b1..55c4d20e3dc 100644 --- a/receiver/otlpreceiver/metrics/otlp_test.go +++ b/receiver/otlpreceiver/metrics/otlp_test.go @@ -27,10 +27,10 @@ import ( "go.opentelemetry.io/collector/consumer" "go.opentelemetry.io/collector/consumer/pdatautil" "go.opentelemetry.io/collector/exporter/exportertest" + "go.opentelemetry.io/collector/internal/data" collectormetrics "go.opentelemetry.io/collector/internal/data/opentelemetry-proto-gen/collector/metrics/v1" otlpcommon "go.opentelemetry.io/collector/internal/data/opentelemetry-proto-gen/common/v1" - otlpmetrics "go.opentelemetry.io/collector/internal/data/opentelemetry-proto-gen/metrics/v1old" - "go.opentelemetry.io/collector/internal/dataold" + otlpmetrics "go.opentelemetry.io/collector/internal/data/opentelemetry-proto-gen/metrics/v1" "go.opentelemetry.io/collector/obsreport" "go.opentelemetry.io/collector/testutil" ) @@ -60,34 +60,37 @@ func TestExport(t *testing.T) { { Metrics: []*otlpmetrics.Metric{ { - MetricDescriptor: &otlpmetrics.MetricDescriptor{ - Name: "mymetric", - Description: "My metric", - Unit: "ms", - Type: otlpmetrics.MetricDescriptor_MONOTONIC_INT64, - }, - Int64DataPoints: []*otlpmetrics.Int64DataPoint{ - { - Labels: []*otlpcommon.StringKeyValue{ + Name: "mymetric", + Description: "My metric", + Unit: "ms", + Data: &otlpmetrics.Metric_IntSum{ + IntSum: &otlpmetrics.IntSum{ + IsMonotonic: true, + AggregationTemporality: otlpmetrics.AggregationTemporality_AGGREGATION_TEMPORALITY_CUMULATIVE, + DataPoints: []*otlpmetrics.IntDataPoint{ { - Key: "key1", - Value: "value1", + Labels: []*otlpcommon.StringKeyValue{ + { + Key: "key1", + Value: "value1", + }, + }, + StartTimeUnixNano: unixnanos1, + TimeUnixNano: unixnanos2, + Value: 123, }, - }, - StartTimeUnixNano: unixnanos1, - TimeUnixNano: unixnanos2, - Value: 123, - }, - { - Labels: []*otlpcommon.StringKeyValue{ { - Key: "key2", - Value: "value2", + Labels: []*otlpcommon.StringKeyValue{ + { + Key: "key2", + Value: "value2", + }, + }, + StartTimeUnixNano: unixnanos1, + TimeUnixNano: unixnanos2, + Value: 456, }, }, - StartTimeUnixNano: unixnanos1, - TimeUnixNano: unixnanos2, - Value: 456, }, }, }, @@ -99,7 +102,7 @@ func TestExport(t *testing.T) { // Keep metric data to compare the test result against it // Clone needed because OTLP proto XXX_ fields are altered in the GRPC downstream - metricData := dataold.MetricDataFromOtlp(resourceMetrics).Clone() + metricData := data.MetricDataFromOtlp(resourceMetrics).Clone() req := &collectormetrics.ExportMetricsServiceRequest{ ResourceMetrics: resourceMetrics, @@ -114,7 +117,7 @@ func TestExport(t *testing.T) { require.Equal(t, 1, len(metricSink.AllMetrics()), "unexpected length: %v", len(metricSink.AllMetrics())) - assert.EqualValues(t, metricData, pdatautil.MetricsToOldInternalMetrics(metricSink.AllMetrics()[0])) + assert.EqualValues(t, metricData, pdatautil.MetricsToInternalMetrics(metricSink.AllMetrics()[0])) } func TestExport_EmptyRequest(t *testing.T) { @@ -153,18 +156,21 @@ func TestExport_ErrorConsumer(t *testing.T) { { Metrics: []*otlpmetrics.Metric{ { - MetricDescriptor: &otlpmetrics.MetricDescriptor{ - Name: "mymetric", - Description: "My metric", - Unit: "ms", - Type: otlpmetrics.MetricDescriptor_MONOTONIC_INT64, - }, - Int64DataPoints: []*otlpmetrics.Int64DataPoint{ - { - Value: 123, - }, - { - Value: 456, + Name: "mymetric", + Description: "My metric", + Unit: "ms", + Data: &otlpmetrics.Metric_IntSum{ + IntSum: &otlpmetrics.IntSum{ + IsMonotonic: true, + AggregationTemporality: otlpmetrics.AggregationTemporality_AGGREGATION_TEMPORALITY_CUMULATIVE, + DataPoints: []*otlpmetrics.IntDataPoint{ + { + Value: 123, + }, + { + Value: 456, + }, + }, }, }, },