Add a 'save failed by service' metric (#1064)

cmd/collector/app/metrics.go

@@ -41,10 +41,11 @@ type SpanProcessorMetrics struct { //TODO - initialize metrics in the traditiona
 	QueueLength metrics.Gauge
 	// ErrorBusy counts number of return ErrServerBusy
 	ErrorBusy metrics.Counter
-	// SavedBySvc contains span and trace counts by service
-	SavedBySvc   metricsBySvc  // spans actually saved
-	serviceNames metrics.Gauge // total number of unique service name metrics reported by this collector
-	spanCounts   map[string]CountsBySpanType
+	// SavedOkBySvc contains span and trace counts by service
+	SavedOkBySvc  metricsBySvc  // spans actually saved
+	SavedErrBySvc metricsBySvc  // spans failed to save
+	serviceNames  metrics.Gauge // total number of unique service name metrics reported by this collector
+	spanCounts    map[string]CountsBySpanType
 }
 
 type countsBySvc struct {
@@ -86,7 +87,8 @@ func NewSpanProcessorMetrics(serviceMetrics metrics.Factory, hostMetrics metrics
 		BatchSize:      hostMetrics.Gauge("batch-size", nil),
 		QueueLength:    hostMetrics.Gauge("queue-length", nil),
 		ErrorBusy:      hostMetrics.Counter("error.busy", nil),
-		SavedBySvc:     newMetricsBySvc(serviceMetrics, "saved-by-svc"),
+		SavedOkBySvc:   newMetricsBySvc(serviceMetrics.Namespace("", map[string]string{"result": "ok"}), "saved-by-svc"),
+		SavedErrBySvc:  newMetricsBySvc(serviceMetrics.Namespace("", map[string]string{"result": "err"}), "saved-by-svc"),
 		spanCounts:     spanCounts,
 		serviceNames:   hostMetrics.Gauge("spans.serviceNames", nil),
 	}

cmd/collector/app/span_processor.go

@@ -100,10 +100,11 @@ func (sp *spanProcessor) saveSpan(span *model.Span) {
 	startTime := time.Now()
 	if err := sp.spanWriter.WriteSpan(span); err != nil {
 		sp.logger.Error("Failed to save span", zap.Error(err))
+		sp.metrics.SavedErrBySvc.ReportServiceNameForSpan(span)
 	} else {
 		sp.logger.Debug("Span written to the storage by the collector",
 			zap.Stringer("trace-id", span.TraceID), zap.Stringer("span-id", span.SpanID))
-		sp.metrics.SavedBySvc.ReportServiceNameForSpan(span)
+		sp.metrics.SavedOkBySvc.ReportServiceNameForSpan(span)
 	}
 	sp.metrics.SaveLatency.Record(time.Since(startTime))
 }

cmd/collector/app/span_processor_test.go

@@ -228,8 +228,11 @@ func TestSpanProcessorErrors(t *testing.T) {
 	w := &fakeSpanWriter{
 		err: fmt.Errorf("some-error"),
 	}
+	mb := metrics.NewLocalFactory(time.Hour)
+	serviceMetrics := mb.Namespace("service", nil)
 	p := NewSpanProcessor(w,
 		Options.Logger(logger),
+		Options.ServiceMetrics(serviceMetrics),
 	).(*spanProcessor)
 
 	res, err := p.ProcessSpans([]*model.Span{
@@ -250,6 +253,11 @@ func TestSpanProcessorErrors(t *testing.T) {
 		"msg":   "Failed to save span",
 		"error": "some-error",
 	}, logBuf.JSONLine(0))
+
+	expected := []metricsTest.ExpectedMetric{{
+		Name: "service.spans.saved-by-svc|debug=false|result=err|svc=x", Value: 1,
+	}}
+	metricsTest.AssertCounterMetrics(t, mb, expected...)
 }
 
 type blockingWriter struct {