From 1461ce69d328055ca156b02ef9661c49ec99c7a8 Mon Sep 17 00:00:00 2001
From: arnikola <artem.nikol.usa@hotmail.com>
Date: Wed, 30 Oct 2019 09:07:02 -0400
Subject: [PATCH] [query] Add latency metrics to remote reads (#2027)

---
 src/query/api/v1/handler/prometheus/native/read.go | 5 +++--
 src/query/api/v1/handler/prometheus/remote/read.go | 8 +++++++-
 2 files changed, 10 insertions(+), 3 deletions(-)

diff --git a/src/query/api/v1/handler/prometheus/native/read.go b/src/query/api/v1/handler/prometheus/native/read.go
index 994fea64ab..a4aa942a65 100644
--- a/src/query/api/v1/handler/prometheus/native/read.go
+++ b/src/query/api/v1/handler/prometheus/native/read.go
@@ -117,12 +117,14 @@ func NewPromReadHandler(
 	keepNans bool,
 	instrumentOpts instrument.Options,
 ) *PromReadHandler {
+	taggedScope := instrumentOpts.MetricsScope().
+		Tagged(map[string]string{"handler": "native-read"})
 	h := &PromReadHandler{
 		engine:              engine,
 		fetchOptionsBuilder: fetchOptionsBuilder,
 		tagOpts:             tagOpts,
 		limitsCfg:           limitsCfg,
-		promReadMetrics:     newPromReadMetrics(instrumentOpts.MetricsScope()),
+		promReadMetrics:     newPromReadMetrics(taggedScope),
 		timeoutOps:          timeoutOpts,
 		keepNans:            keepNans,
 		instrumentOpts:      instrumentOpts,
@@ -134,7 +136,6 @@ func NewPromReadHandler(
 
 func (h *PromReadHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 	timer := h.promReadMetrics.fetchTimerSuccess.Start()
-
 	fetchOpts, rErr := h.fetchOptionsBuilder.NewFetchOptions(r)
 	if rErr != nil {
 		xhttp.Error(w, rErr.Inner(), rErr.Code())
diff --git a/src/query/api/v1/handler/prometheus/remote/read.go b/src/query/api/v1/handler/prometheus/remote/read.go
index c36cc5dc35..a4280bd4ff 100644
--- a/src/query/api/v1/handler/prometheus/remote/read.go
+++ b/src/query/api/v1/handler/prometheus/remote/read.go
@@ -70,9 +70,11 @@ func NewPromReadHandler(
 	keepEmpty bool,
 	instrumentOpts instrument.Options,
 ) http.Handler {
+	taggedScope := instrumentOpts.MetricsScope().
+		Tagged(map[string]string{"handler": "remote-read"})
 	return &PromReadHandler{
 		engine:              engine,
-		promReadMetrics:     newPromReadMetrics(instrumentOpts.MetricsScope()),
+		promReadMetrics:     newPromReadMetrics(taggedScope),
 		timeoutOpts:         timeoutOpts,
 		fetchOptionsBuilder: fetchOptionsBuilder,
 		keepEmpty:           keepEmpty,
@@ -84,6 +86,7 @@ type promReadMetrics struct {
 	fetchSuccess      tally.Counter
 	fetchErrorsServer tally.Counter
 	fetchErrorsClient tally.Counter
+	fetchTimerSuccess tally.Timer
 }
 
 func newPromReadMetrics(scope tally.Scope) promReadMetrics {
@@ -94,10 +97,12 @@ func newPromReadMetrics(scope tally.Scope) promReadMetrics {
 			Counter("fetch.errors"),
 		fetchErrorsClient: scope.Tagged(map[string]string{"code": "4XX"}).
 			Counter("fetch.errors"),
+		fetchTimerSuccess: scope.Timer("fetch.success.latency"),
 	}
 }
 
 func (h *PromReadHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	timer := h.promReadMetrics.fetchTimerSuccess.Start()
 	ctx := context.WithValue(r.Context(), handler.HeaderKey, r.Header)
 	logger := logging.WithContext(ctx, h.instrumentOpts)
 	req, rErr := h.parseRequest(r)
@@ -152,6 +157,7 @@ func (h *PromReadHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
+	timer.Stop()
 	h.promReadMetrics.fetchSuccess.Inc(1)
 }