From 981cefbd2a512362013b2d01729a5162f8d0a5b5 Mon Sep 17 00:00:00 2001 From: Jacob Shufro Date: Wed, 20 Nov 2024 17:34:48 -0500 Subject: [PATCH] Add backoff to validator balance queries for treegen --- shared/services/bc-manager.go | 11 ++++ shared/services/beacon/client.go | 1 + .../services/beacon/client/std-http-client.go | 66 ++++++++++++++++++- shared/services/beacon/client/types.go | 4 +- .../services/rewards/generator-impl-v9-v10.go | 8 +-- shared/services/rewards/test/beacon.go | 8 +++ shared/services/rewards/types.go | 2 +- 7 files changed, 91 insertions(+), 9 deletions(-) diff --git a/shared/services/bc-manager.go b/shared/services/bc-manager.go index d393fcce1..fda19834b 100644 --- a/shared/services/bc-manager.go +++ b/shared/services/bc-manager.go @@ -309,6 +309,17 @@ func (m *BeaconClientManager) ChangeWithdrawalCredentials(validatorIndex string, return nil } +// Get the validator balances for a set of validators at a given slot, with backoff. +func (m *BeaconClientManager) GetValidatorBalancesSafe(indices []string, opts *beacon.ValidatorStatusOptions) (map[string]*big.Int, error) { + result, err := m.runFunction1(func(client beacon.Client) (interface{}, error) { + return client.GetValidatorBalancesSafe(indices, opts) + }) + if err != nil { + return nil, err + } + return result.(map[string]*big.Int), nil +} + // Get the validator balances for a set of validators at a given slot func (m *BeaconClientManager) GetValidatorBalances(indices []string, opts *beacon.ValidatorStatusOptions) (map[string]*big.Int, error) { result, err := m.runFunction1(func(client beacon.Client) (interface{}, error) { diff --git a/shared/services/beacon/client.go b/shared/services/beacon/client.go index 9120e0c70..353bb7c15 100644 --- a/shared/services/beacon/client.go +++ b/shared/services/beacon/client.go @@ -145,6 +145,7 @@ type Client interface { GetValidatorSyncDuties(indices []string, epoch uint64) (map[string]bool, error) GetValidatorProposerDuties(indices []string, epoch uint64) (map[string]uint64, error) GetValidatorBalances(indices []string, opts *ValidatorStatusOptions) (map[string]*big.Int, error) + GetValidatorBalancesSafe(indices []string, opts *ValidatorStatusOptions) (map[string]*big.Int, error) GetDomainData(domainType []byte, epoch uint64, useGenesisFork bool) ([]byte, error) ExitValidator(validatorIndex string, epoch uint64, signature types.ValidatorSignature) error Close() error diff --git a/shared/services/beacon/client/std-http-client.go b/shared/services/beacon/client/std-http-client.go index d73a25a1e..22a1974d5 100644 --- a/shared/services/beacon/client/std-http-client.go +++ b/shared/services/beacon/client/std-http-client.go @@ -10,6 +10,7 @@ import ( "strconv" "strings" "sync" + "sync/atomic" "time" "github.com/ethereum/go-ethereum/common" @@ -92,9 +93,18 @@ func (c *StandardHttpClient) GetSyncStatus() (beacon.SyncStatus, error) { } +var eth2ConfigCache atomic.Pointer[beacon.Eth2Config] + // Get the eth2 config +// cache it for future requests func (c *StandardHttpClient) GetEth2Config() (beacon.Eth2Config, error) { + // Check the cache + cached := eth2ConfigCache.Load() + if cached != nil { + return *cached, nil + } + // Data var wg errgroup.Group var eth2Config Eth2ConfigResponse @@ -119,8 +129,8 @@ func (c *StandardHttpClient) GetEth2Config() (beacon.Eth2Config, error) { return beacon.Eth2Config{}, err } - // Return response - return beacon.Eth2Config{ + // Save the result + out := beacon.Eth2Config{ GenesisForkVersion: genesis.Data.GenesisForkVersion, GenesisValidatorsRoot: genesis.Data.GenesisValidatorsRoot, GenesisEpoch: 0, @@ -129,8 +139,11 @@ func (c *StandardHttpClient) GetEth2Config() (beacon.Eth2Config, error) { SlotsPerEpoch: uint64(eth2Config.Data.SlotsPerEpoch), SecondsPerEpoch: uint64(eth2Config.Data.SecondsPerSlot * eth2Config.Data.SlotsPerEpoch), EpochsPerSyncCommitteePeriod: uint64(eth2Config.Data.EpochsPerSyncCommitteePeriod), - }, nil + } + eth2ConfigCache.Store(&out) + // Return + return out, nil } // Get the eth2 deposit contract info @@ -302,6 +315,53 @@ func (c *StandardHttpClient) GetValidatorBalances(indices []string, opts *beacon return data, nil } +// GetValidatorBalancesSafe returns the balances of the validators +// In order to avoid thrashing the bn, when opts.Slot is provided, +// we will preflight the balance query with a sync query, and ensure that the +// bn has not entered optimistic sync due to being unable to provide forkchoice updates, +// and that the current head is a recent slot. +func (c *StandardHttpClient) GetValidatorBalancesSafe(indices []string, opts *beacon.ValidatorStatusOptions) (map[string]*big.Int, error) { + beaconConfig, err := c.GetEth2Config() + if err != nil { + return nil, err + } + // Check the current head + safe := false + for i := 0; i < 30; i++ { + syncStatus, err := c.getSyncStatus() + if err != nil { + // If we get an error, wait and try again + time.Sleep(1 * time.Second) + continue + } + if syncStatus.Data.IsSyncing { + // If the bn is still syncing, wait and try again + time.Sleep(1 * time.Second) + continue + } + if syncStatus.Data.ELOffline { + // If the bn is offline, wait and try again + time.Sleep(1 * time.Second) + continue + } + // Check that the head is no more than 2 slots behind the current time. + if beaconConfig.GetSlotTime(uint64(syncStatus.Data.HeadSlot)).Add(2 * time.Second * time.Duration(beaconConfig.SecondsPerSlot)).Before(time.Now()) { + // If the head is too far behind, wait and try again + time.Sleep(1 * time.Second) + continue + } + + safe = true + break + } + if !safe { + return nil, fmt.Errorf("bn is not in sync after 30 seconds") + } + + // Get the balances + return c.GetValidatorBalances(indices, opts) +} + // Get multiple validators' statuses func (c *StandardHttpClient) GetValidatorStatuses(pubkeys []types.ValidatorPubkey, opts *beacon.ValidatorStatusOptions) (map[types.ValidatorPubkey]beacon.ValidatorStatus, error) { diff --git a/shared/services/beacon/client/types.go b/shared/services/beacon/client/types.go index 75e72263c..00882fe0a 100644 --- a/shared/services/beacon/client/types.go +++ b/shared/services/beacon/client/types.go @@ -32,9 +32,11 @@ type BLSToExecutionChangeRequest struct { // Response types type SyncStatusResponse struct { Data struct { - IsSyncing bool `json:"is_syncing"` HeadSlot uinteger `json:"head_slot"` SyncDistance uinteger `json:"sync_distance"` + IsSyncing bool `json:"is_syncing"` + IsOptimistic bool `json:"is_optimistic"` + ELOffline bool `json:"el_offline"` } `json:"data"` } type Eth2ConfigResponse struct { diff --git a/shared/services/rewards/generator-impl-v9-v10.go b/shared/services/rewards/generator-impl-v9-v10.go index a841671d9..141ca7e7f 100644 --- a/shared/services/rewards/generator-impl-v9-v10.go +++ b/shared/services/rewards/generator-impl-v9-v10.go @@ -820,7 +820,7 @@ func (r *treeGeneratorImpl_v9_v10) getValidatorBalancesAtStartAndEnd() error { r.log.Printlnf("%s Getting %d validator balances at start and end", r.logPrefix, len(indices)) - validatorBalancesAtStart, err := r.bc.GetValidatorBalances(indices, &beacon.ValidatorStatusOptions{ + validatorBalancesAtStart, err := r.bc.GetValidatorBalancesSafe(indices, &beacon.ValidatorStatusOptions{ Slot: &r.rewardsFile.ConsensusStartBlock, }) if err != nil { @@ -830,7 +830,7 @@ func (r *treeGeneratorImpl_v9_v10) getValidatorBalancesAtStartAndEnd() error { r.validatorBalancesAtStart[r.validatorIndexMap[index].Address] = balance } - validatorBalancesAtEnd, err := r.bc.GetValidatorBalances(indices, &beacon.ValidatorStatusOptions{ + validatorBalancesAtEnd, err := r.bc.GetValidatorBalancesSafe(indices, &beacon.ValidatorStatusOptions{ Slot: &r.rewardsFile.ConsensusEndBlock, }) if err != nil { @@ -917,7 +917,7 @@ func (r *treeGeneratorImpl_v9_v10) getValidatorBalancesAtStartAndEnd() error { r.log.Printlnf("%s On slot %d of %d (%.2f%%) - %.2f seconds per slot", r.logPrefix, i, total, float64(i)/float64(total)*100.0, secondsPerSlot) } i++ - balances, err := r.bc.GetValidatorBalances(validatorIndices, &beacon.ValidatorStatusOptions{ + balances, err := r.bc.GetValidatorBalancesSafe(validatorIndices, &beacon.ValidatorStatusOptions{ Slot: &slot, }) if err != nil { @@ -934,7 +934,7 @@ func (r *treeGeneratorImpl_v9_v10) getValidatorBalancesAtStartAndEnd() error { r.log.Printlnf("%s On slot %d of %d (%.2f%%) - %.2f seconds per slot", r.logPrefix, i, total, float64(i)/float64(total)*100.0, secondsPerSlot) } i++ - balances, err := r.bc.GetValidatorBalances(validatorIndices, &beacon.ValidatorStatusOptions{ + balances, err := r.bc.GetValidatorBalancesSafe(validatorIndices, &beacon.ValidatorStatusOptions{ Slot: &slot, }) if err != nil { diff --git a/shared/services/rewards/test/beacon.go b/shared/services/rewards/test/beacon.go index 0fa4e69b1..0b801899b 100644 --- a/shared/services/rewards/test/beacon.go +++ b/shared/services/rewards/test/beacon.go @@ -422,6 +422,14 @@ func (bc *MockBeaconClient) SetCustomBalance(index string, balance *big.Int, slo }{balance, slot}) } +// GetValidatorBalancesSafe returns the balances of the validators +// The mock doesn't need to worry about thrashing the bn, since there is none. +func (bc *MockBeaconClient) GetValidatorBalancesSafe(indices []string, opts *beacon.ValidatorStatusOptions) (map[string]*big.Int, error) { + + // Get the balances + return bc.GetValidatorBalances(indices, opts) +} + func (bc *MockBeaconClient) GetValidatorBalances(indices []string, opts *beacon.ValidatorStatusOptions) (map[string]*big.Int, error) { out := make(map[string]*big.Int) for _, index := range indices { diff --git a/shared/services/rewards/types.go b/shared/services/rewards/types.go index b1d7e4020..76dd4a7db 100644 --- a/shared/services/rewards/types.go +++ b/shared/services/rewards/types.go @@ -50,7 +50,7 @@ type RewardsBeaconClient interface { GetBeaconBlock(slot string) (beacon.BeaconBlock, bool, error) GetCommitteesForEpoch(epoch *uint64) (beacon.Committees, error) GetAttestations(slot string) ([]beacon.AttestationInfo, bool, error) - GetValidatorBalances(indices []string, opts *beacon.ValidatorStatusOptions) (map[string]*big.Int, error) + GetValidatorBalancesSafe(indices []string, opts *beacon.ValidatorStatusOptions) (map[string]*big.Int, error) GetEth2Config() (beacon.Eth2Config, error) GetBeaconHead() (beacon.BeaconHead, error) }