Add requests in flight metric

stellar · Feb 29, 2024 · 3014772 · 3014772
1 parent d22d789
commit 3014772
Show file tree

Hide file tree

Showing 2 changed files with 20 additions and 3 deletions.
diff --git a/services/horizon/internal/httpx/middleware.go b/services/horizon/internal/httpx/middleware.go
@@ -78,11 +78,20 @@ func loggerMiddleware(serverMetrics *ServerMetrics) func(next http.Handler) http
 			// is reset before sending the first event no Content-Type header is sent in a response.
 			acceptHeader := r.Header.Get("Accept")
 			streaming := strings.Contains(acceptHeader, render.MimeEventStream)
+			route := supportHttp.GetChiRoutePattern(r)
+
+			inFlightLabels := prometheus.Labels{
+				"route":     route,
+				"streaming": strconv.FormatBool(streaming),
+				"method":    r.Method,
+			}
+			serverMetrics.RequestsInFlightGauge.With(inFlightLabels).Inc()
+			defer serverMetrics.RequestsInFlightGauge.With(inFlightLabels).Dec()
 
 			then := time.Now()
 			next.ServeHTTP(mw, r.WithContext(ctx))
 			duration := time.Since(then)
-			logEndOfRequest(ctx, r, serverMetrics.RequestDurationSummary, duration, mw, streaming)
+			logEndOfRequest(ctx, r, route, serverMetrics.RequestDurationSummary, duration, mw, streaming)
 		})
 	}
 }
@@ -129,8 +138,7 @@ func getClientData(r *http.Request, headerName string) string {
 	return value
 }
 
-func logEndOfRequest(ctx context.Context, r *http.Request, requestDurationSummary *prometheus.SummaryVec, duration time.Duration, mw middleware.WrapResponseWriter, streaming bool) {
-	route := supportHttp.GetChiRoutePattern(r)
+func logEndOfRequest(ctx context.Context, r *http.Request, route string, requestDurationSummary *prometheus.SummaryVec, duration time.Duration, mw middleware.WrapResponseWriter, streaming bool) {
 
 	referer := r.Referer()
 	if referer == "" {

diff --git a/services/horizon/internal/httpx/server.go b/services/horizon/internal/httpx/server.go
@@ -23,6 +23,7 @@ import (
 type ServerMetrics struct {
 	RequestDurationSummary  *prometheus.SummaryVec
 	ReplicaLagErrorsCounter prometheus.Counter
+	RequestsInFlightGauge   *prometheus.GaugeVec
 }
 
 type TLSConfig struct {
@@ -71,6 +72,13 @@ func NewServer(serverConfig ServerConfig, routerConfig RouterConfig, ledgerState
 			},
 			[]string{"status", "route", "streaming", "method"},
 		),
+		RequestsInFlightGauge: prometheus.NewGaugeVec(
+			prometheus.GaugeOpts{
+				Namespace: "horizon", Subsystem: "http", Name: "requests_in_flight",
+				Help: "HTTP requests in flight",
+			},
+			[]string{"route", "streaming", "method"},
+		),
 		ReplicaLagErrorsCounter: prometheus.NewCounter(
 			prometheus.CounterOpts{
 				Namespace: "horizon", Subsystem: "http", Name: "replica_lag_errors_count",
@@ -109,6 +117,7 @@ func NewServer(serverConfig ServerConfig, routerConfig RouterConfig, ledgerState
 func (s *Server) RegisterMetrics(registry *prometheus.Registry) {
 	registry.MustRegister(s.Metrics.RequestDurationSummary)
 	registry.MustRegister(s.Metrics.ReplicaLagErrorsCounter)
+	registry.MustRegister(s.Metrics.RequestsInFlightGauge)
 }
 
 func (s *Server) Serve() error {