From 68b39bd670c1cff31cecd1c59e38bc4ce72aafeb Mon Sep 17 00:00:00 2001 From: Bartek Nowotarski Date: Thu, 21 Oct 2021 14:35:23 +0200 Subject: [PATCH] services/horizon: Add ledger_fetch_duration_seconds metric (#4016) Add `horizon_ingest_ledger_fetch_duration_seconds` metric indicating time required to fetch a ledger from `LedgerBackend`. We don't track this value outside logging but it can be useful (ex. alerting if ledger backend is slow). --- services/horizon/internal/ingest/fsm.go | 7 +++++-- services/horizon/internal/ingest/main.go | 11 +++++++++++ 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/services/horizon/internal/ingest/fsm.go b/services/horizon/internal/ingest/fsm.go index d610d7f5c7..9078f6a13e 100644 --- a/services/horizon/internal/ingest/fsm.go +++ b/services/horizon/internal/ingest/fsm.go @@ -390,11 +390,14 @@ func (r resumeState) run(s *system) (transition, error) { if err != nil { return start(), errors.Wrap(err, "error getting ledger blocking") } + duration := time.Since(startTime).Seconds() log.WithFields(logpkg.F{ "sequence": ingestLedger, - "duration": time.Since(startTime).Seconds(), + "duration": duration, }).Info("Ledger returned from the backend") + s.Metrics().LedgerFetchDurationSummary.Observe(float64(duration)) + if err = s.historyQ.Begin(); err != nil { return retryResume(r), errors.Wrap(err, "Error starting a transaction") @@ -480,7 +483,7 @@ func (r resumeState) run(s *system) (transition, error) { log.WithError(err).Warn("error updating stellar-core cursor") } - duration := time.Since(startTime).Seconds() + duration = time.Since(startTime).Seconds() s.Metrics().LedgerIngestionDuration.Observe(float64(duration)) // Update stats metrics diff --git a/services/horizon/internal/ingest/main.go b/services/horizon/internal/ingest/main.go index 46ece7c788..31f38e82b7 100644 --- a/services/horizon/internal/ingest/main.go +++ b/services/horizon/internal/ingest/main.go @@ -139,6 +139,10 @@ type Metrics struct { // ProcessorsRunDurationSummary exposes processors run durations. ProcessorsRunDurationSummary *prometheus.SummaryVec + // LedgerFetchDurationSummary exposes a summary of durations required to + // fetch data from ledger backend. + LedgerFetchDurationSummary prometheus.Summary + // CaptiveStellarCoreSynced exposes synced status of Captive Stellar-Core. // 1 if sync, 0 if not synced, -1 if unable to connect or HTTP server disabled. CaptiveStellarCoreSynced prometheus.GaugeFunc @@ -353,6 +357,13 @@ func (s *system) initMetrics() { []string{"name"}, ) + s.metrics.LedgerFetchDurationSummary = prometheus.NewSummary( + prometheus.SummaryOpts{ + Namespace: "horizon", Subsystem: "ingest", Name: "ledger_fetch_duration_seconds", + Help: "duration of fetching ledgers from ledger backend, sliding window = 10m", + }, + ) + s.metrics.CaptiveStellarCoreSynced = prometheus.NewGaugeFunc( prometheus.GaugeOpts{ Namespace: "horizon", Subsystem: "ingest", Name: "captive_stellar_core_synced",