services/horizon: Add ledger_fetch_duration_seconds metric (#4016)

Add `horizon_ingest_ledger_fetch_duration_seconds` metric indicating time
required to fetch a ledger from `LedgerBackend`. We don't track this value
outside logging but it can be useful (ex. alerting if ledger backend is slow).

services/horizon/internal/ingest/fsm.go

@@ -390,11 +390,14 @@ func (r resumeState) run(s *system) (transition, error) {
 	if err != nil {
 		return start(), errors.Wrap(err, "error getting ledger blocking")
 	}
+	duration := time.Since(startTime).Seconds()
 	log.WithFields(logpkg.F{
 		"sequence": ingestLedger,
-		"duration": time.Since(startTime).Seconds(),
+		"duration": duration,
 	}).Info("Ledger returned from the backend")
 
+	s.Metrics().LedgerFetchDurationSummary.Observe(float64(duration))
+
 	if err = s.historyQ.Begin(); err != nil {
 		return retryResume(r),
 			errors.Wrap(err, "Error starting a transaction")
@@ -480,7 +483,7 @@ func (r resumeState) run(s *system) (transition, error) {
 		log.WithError(err).Warn("error updating stellar-core cursor")
 	}
 
-	duration := time.Since(startTime).Seconds()
+	duration = time.Since(startTime).Seconds()
 	s.Metrics().LedgerIngestionDuration.Observe(float64(duration))
 
 	// Update stats metrics

services/horizon/internal/ingest/main.go

@@ -139,6 +139,10 @@ type Metrics struct {
 	// ProcessorsRunDurationSummary exposes processors run durations.
 	ProcessorsRunDurationSummary *prometheus.SummaryVec
 
+	// LedgerFetchDurationSummary exposes a summary of durations required to
+	// fetch data from ledger backend.
+	LedgerFetchDurationSummary prometheus.Summary
+
 	// CaptiveStellarCoreSynced exposes synced status of Captive Stellar-Core.
 	// 1 if sync, 0 if not synced, -1 if unable to connect or HTTP server disabled.
 	CaptiveStellarCoreSynced prometheus.GaugeFunc
@@ -353,6 +357,13 @@ func (s *system) initMetrics() {
 		[]string{"name"},
 	)
 
+	s.metrics.LedgerFetchDurationSummary = prometheus.NewSummary(
+		prometheus.SummaryOpts{
+			Namespace: "horizon", Subsystem: "ingest", Name: "ledger_fetch_duration_seconds",
+			Help: "duration of fetching ledgers from ledger backend, sliding window = 10m",
+		},
+	)
+
 	s.metrics.CaptiveStellarCoreSynced = prometheus.NewGaugeFunc(
 		prometheus.GaugeOpts{
 			Namespace: "horizon", Subsystem: "ingest", Name: "captive_stellar_core_synced",