From eace845acd4e455c77f248526a6633089c380054 Mon Sep 17 00:00:00 2001 From: peter <1674920+peterbitfly@users.noreply.github.com> Date: Thu, 3 Oct 2024 06:38:37 +0000 Subject: [PATCH 1/3] feat(validators): improve performance of validator status filters --- handlers/validators.go | 34 ++++++++++++++++++------------- services/services.go | 45 ------------------------------------------ types/frontend.go | 4 ++-- 3 files changed, 22 insertions(+), 61 deletions(-) diff --git a/handlers/validators.go b/handlers/validators.go index cd61d43900..d33f996308 100644 --- a/handlers/validators.go +++ b/handlers/validators.go @@ -26,30 +26,36 @@ func Validators(w http.ResponseWriter, r *http.Request) { validatorsPageData := types.ValidatorsPageData{} - currentStateCounts := services.LatestValidatorStateCounts() + var currentStateCounts []*types.ValidatorStateCountRow + err := db.ReaderDb.Select(¤tStateCounts, "SELECT status, validator_count FROM validators_status_counts") + if err != nil { + utils.LogError(err, "error retrieving validators state counts", 0, nil) + http.Error(w, "Internal server error", http.StatusInternalServerError) + return + } - for _, state := range *currentStateCounts { - switch state.Name { + for _, status := range currentStateCounts { + switch status.Name { case "pending": - validatorsPageData.PendingCount = state.Count + validatorsPageData.PendingCount = status.Count case "active_online": - validatorsPageData.ActiveOnlineCount = state.Count + validatorsPageData.ActiveOnlineCount = status.Count case "active_offline": - validatorsPageData.ActiveOfflineCount = state.Count + validatorsPageData.ActiveOfflineCount = status.Count case "slashing_online": - validatorsPageData.SlashingOnlineCount = state.Count + validatorsPageData.SlashingOnlineCount = status.Count case "slashing_offline": - validatorsPageData.SlashingOfflineCount = state.Count + validatorsPageData.SlashingOfflineCount = status.Count case "slashed": - validatorsPageData.Slashed = state.Count + validatorsPageData.Slashed = status.Count case "exiting_online": - validatorsPageData.ExitingOnlineCount = state.Count + validatorsPageData.ExitingOnlineCount = status.Count case "exiting_offline": - validatorsPageData.ExitingOfflineCount = state.Count + validatorsPageData.ExitingOfflineCount = status.Count case "exited": - validatorsPageData.VoluntaryExitsCount = state.Count + validatorsPageData.VoluntaryExitsCount = status.Count case "deposited": - validatorsPageData.DepositedCount = state.Count + validatorsPageData.DepositedCount = status.Count } } @@ -352,7 +358,7 @@ func ValidatorsData(w http.ResponseWriter, r *http.Request) { } countFiltered := uint64(0) if dataQuery.StateFilter != "" { - qry = fmt.Sprintf(`SELECT COUNT(*) FROM validators %s`, dataQuery.StateFilter) + qry = fmt.Sprintf(`SELECT SUM(validator_count) FROM validators_status_counts AS validators %s`, dataQuery.StateFilter) err = db.ReaderDb.Get(&countFiltered, qry) if err != nil { utils.LogError(err, "error retrieving validators total count", 0, errFields) diff --git a/services/services.go b/services/services.go index b78f21451e..9dc6ca9f09 100644 --- a/services/services.go +++ b/services/services.go @@ -87,9 +87,6 @@ func Init() { ready.Add(1) go latestExportedStatisticDayUpdater(ready) - ready.Add(1) - go validatorStateCountsUpdater(ready) - if utils.Config.RatelimitUpdater.Enabled { go ratelimit.DBUpdater() } @@ -1769,45 +1766,3 @@ func LatestExportedStatisticDay() (uint64, error) { } return wanted, nil } - -func validatorStateCountsUpdater(wg *sync.WaitGroup) { - firstRun := true - - for { - var currentStateCounts []types.ValidatorStateCountRow - qry := "SELECT status AS statename, COUNT(*) AS statecount FROM validators GROUP BY status" - err := db.ReaderDb.Select(¤tStateCounts, qry) - - if err != nil { - logger.Errorf("error retrieving validator state counts from the database: %v", err) - - if err.Error() == "sql: database is closed" { - logger.Fatalf("error retrieving validator state counts from the database: %v", err) - } - } else { - cacheKey := fmt.Sprintf("%d:frontend:validator_state_counts", utils.Config.Chain.ClConfig.DepositChainID) - err := cache.TieredCache.Set(cacheKey, currentStateCounts, utils.Day) - if err != nil { - logger.Errorf("error caching validator state counts: %v", err) - } - if firstRun { - logger.Info("initialized validator state counts updater") - wg.Done() - firstRun = false - } - } - ReportStatus("validatorStateCountsUpdater", "Running", nil) - time.Sleep(time.Minute * 60) - } -} - -func LatestValidatorStateCounts() *[]types.ValidatorStateCountRow { - wanted := []types.ValidatorStateCountRow{} - cacheKey := fmt.Sprintf("%d:frontend:validator_state_counts", utils.Config.Chain.ClConfig.DepositChainID) - if wanted, err := cache.TieredCache.GetWithLocalTimeout(cacheKey, time.Minute, &wanted); err == nil { - return wanted.(*[]types.ValidatorStateCountRow) - } else { - logger.Errorf("error retrieving validator state count data from cache: %v", err) - } - return &wanted -} diff --git a/types/frontend.go b/types/frontend.go index 88df7b7023..6216db2f86 100644 --- a/types/frontend.go +++ b/types/frontend.go @@ -617,6 +617,6 @@ type SearchValidatorsByEth1Result []struct { } type ValidatorStateCountRow struct { - Name string `db:"statename"` - Count uint64 `db:"statecount"` + Name string `db:"status"` + Count uint64 `db:"validator_count"` } From 6f290f7202c92b9eea7068efdda3dd32567932fb Mon Sep 17 00:00:00 2001 From: peter <1674920+peterbitfly@users.noreply.github.com> Date: Thu, 3 Oct 2024 07:15:24 +0000 Subject: [PATCH 2/3] feat(exporter): write validator status counts to dedicated table --- db/db.go | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/db/db.go b/db/db.go index 9ca297c989..2217eb7f8c 100644 --- a/db/db.go +++ b/db/db.go @@ -989,6 +989,8 @@ func SaveValidators(epoch uint64, validators []*types.Validator, client rpc.Clie return fmt.Errorf("error preparing insert validator statement: %w", err) } + validatorStatusCounts := make(map[string]int) + updates := 0 for _, v := range validators { @@ -1031,6 +1033,7 @@ func SaveValidators(epoch uint64, validators []*types.Validator, client rpc.Clie if err != nil { logger.Errorf("error saving new validator %v: %v", v.Index, err) } + validatorStatusCounts[v.Status]++ } else { // status = // CASE @@ -1071,6 +1074,7 @@ func SaveValidators(epoch uint64, validators []*types.Validator, client rpc.Clie v.Status = "active_online" } + validatorStatusCounts[v.Status]++ if c.Status != v.Status { logger.Tracef("Status changed for validator %v from %v to %v", v.Index, c.Status, v.Status) // logger.Tracef("v.ActivationEpoch %v, latestEpoch %v, lastAttestationSlots[v.Index] %v, thresholdSlot %v", v.ActivationEpoch, latestEpoch, lastAttestationSlots[v.Index], thresholdSlot) @@ -1199,6 +1203,20 @@ func SaveValidators(epoch uint64, validators []*types.Validator, client rpc.Clie logger.Infof("updating validator activation epoch balance completed, took %v", time.Since(s)) + logger.Infof("updating validator status counts") + s = time.Now() + _, err = tx.Exec("TRUNCATE TABLE validators_status_counts;") + if err != nil { + return fmt.Errorf("error truncating validators_status_counts table: %w", err) + } + for status, count := range validatorStatusCounts { + _, err = tx.Exec("INSERT INTO validators_status_counts (status, validator_count) VALUES ($1, $2);", status, count) + if err != nil { + return fmt.Errorf("error updating validator status counts: %w", err) + } + } + logger.Infof("updating validator status counts completed, took %v", time.Since(s)) + s = time.Now() _, err = tx.Exec("ANALYZE (SKIP_LOCKED) validators;") if err != nil { From 838bf05162aef05512bf0843d82786e609d78700 Mon Sep 17 00:00:00 2001 From: peter <1674920+peterbitfly@users.noreply.github.com> Date: Thu, 3 Oct 2024 07:30:40 +0000 Subject: [PATCH 3/3] feat(exporter): add migration file --- .../20241003062452_validator_status_counts.sql | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 db/migrations/20241003062452_validator_status_counts.sql diff --git a/db/migrations/20241003062452_validator_status_counts.sql b/db/migrations/20241003062452_validator_status_counts.sql new file mode 100644 index 0000000000..dbb6d2ebc3 --- /dev/null +++ b/db/migrations/20241003062452_validator_status_counts.sql @@ -0,0 +1,11 @@ +-- +goose Up +-- +goose StatementBegin +SELECT 'up SQL query'; +CREATE TABLE IF NOT EXISTS validators_status_counts (status varchar(20) primary key, validator_count int not null); +-- +goose StatementEnd + +-- +goose Down +-- +goose StatementBegin +SELECT 'down SQL query'; +DROP TABLE IF EXISTS validators_status_counts; +-- +goose StatementEnd