Add backoff to validator balance queries for treegen

shared/services/bc-manager.go

@@ -309,6 +309,17 @@ func (m *BeaconClientManager) ChangeWithdrawalCredentials(validatorIndex string,
 	return nil
 }
 
+// Get the validator balances for a set of validators at a given slot, with backoff.
+func (m *BeaconClientManager) GetValidatorBalancesSafe(indices []string, opts *beacon.ValidatorStatusOptions) (map[string]*big.Int, error) {
+	result, err := m.runFunction1(func(client beacon.Client) (interface{}, error) {
+		return client.GetValidatorBalancesSafe(indices, opts)
+	})
+	if err != nil {
+		return nil, err
+	}
+	return result.(map[string]*big.Int), nil
+}
+
 // Get the validator balances for a set of validators at a given slot
 func (m *BeaconClientManager) GetValidatorBalances(indices []string, opts *beacon.ValidatorStatusOptions) (map[string]*big.Int, error) {
 	result, err := m.runFunction1(func(client beacon.Client) (interface{}, error) {

shared/services/beacon/client.go

@@ -145,6 +145,7 @@ type Client interface {
 	GetValidatorSyncDuties(indices []string, epoch uint64) (map[string]bool, error)
 	GetValidatorProposerDuties(indices []string, epoch uint64) (map[string]uint64, error)
 	GetValidatorBalances(indices []string, opts *ValidatorStatusOptions) (map[string]*big.Int, error)
+	GetValidatorBalancesSafe(indices []string, opts *ValidatorStatusOptions) (map[string]*big.Int, error)
 	GetDomainData(domainType []byte, epoch uint64, useGenesisFork bool) ([]byte, error)
 	ExitValidator(validatorIndex string, epoch uint64, signature types.ValidatorSignature) error
 	Close() error

shared/services/beacon/client/std-http-client.go

@@ -10,6 +10,7 @@ import (
 	"strconv"
 	"strings"
 	"sync"
+	"sync/atomic"
 	"time"
 
 	"github.com/ethereum/go-ethereum/common"
@@ -92,9 +93,18 @@ func (c *StandardHttpClient) GetSyncStatus() (beacon.SyncStatus, error) {
 
 }
 
+var eth2ConfigCache atomic.Pointer[beacon.Eth2Config]
+
 // Get the eth2 config
+// cache it for future requests
 func (c *StandardHttpClient) GetEth2Config() (beacon.Eth2Config, error) {
 
+	// Check the cache
+	cached := eth2ConfigCache.Load()
+	if cached != nil {
+		return *cached, nil
+	}
+
 	// Data
 	var wg errgroup.Group
 	var eth2Config Eth2ConfigResponse
@@ -119,8 +129,8 @@ func (c *StandardHttpClient) GetEth2Config() (beacon.Eth2Config, error) {
 		return beacon.Eth2Config{}, err
 	}
 
-	// Return response
-	return beacon.Eth2Config{
+	// Save the result
+	out := beacon.Eth2Config{
 		GenesisForkVersion:           genesis.Data.GenesisForkVersion,
 		GenesisValidatorsRoot:        genesis.Data.GenesisValidatorsRoot,
 		GenesisEpoch:                 0,
@@ -129,8 +139,11 @@ func (c *StandardHttpClient) GetEth2Config() (beacon.Eth2Config, error) {
 		SlotsPerEpoch:                uint64(eth2Config.Data.SlotsPerEpoch),
 		SecondsPerEpoch:              uint64(eth2Config.Data.SecondsPerSlot * eth2Config.Data.SlotsPerEpoch),
 		EpochsPerSyncCommitteePeriod: uint64(eth2Config.Data.EpochsPerSyncCommitteePeriod),
-	}, nil
+	}
+	eth2ConfigCache.Store(&out)
 
+	// Return
+	return out, nil
 }
 
 // Get the eth2 deposit contract info
@@ -302,6 +315,53 @@ func (c *StandardHttpClient) GetValidatorBalances(indices []string, opts *beacon
 	return data, nil
 }
 
+// GetValidatorBalancesSafe returns the balances of the validators
+// In order to avoid thrashing the bn, when opts.Slot is provided,
+// we will preflight the balance query with a sync query, and ensure that the
+// bn has not entered optimistic sync due to being unable to provide forkchoice updates,
+// and that the current head is a recent slot.
+func (c *StandardHttpClient) GetValidatorBalancesSafe(indices []string, opts *beacon.ValidatorStatusOptions) (map[string]*big.Int, error) {
+	beaconConfig, err := c.GetEth2Config()
+	if err != nil {
+		return nil, err
+	}
+	// Check the current head
+	safe := false
+	for i := 0; i < 30; i++ {
+		syncStatus, err := c.getSyncStatus()
+		if err != nil {
+			// If we get an error, wait and try again
+			time.Sleep(1 * time.Second)
+			continue
+		}
+		if syncStatus.Data.IsSyncing {
+			// If the bn is still syncing, wait and try again
+			time.Sleep(1 * time.Second)
+			continue
+		}
+		if syncStatus.Data.ELOffline {
+			// If the bn is offline, wait and try again
+			time.Sleep(1 * time.Second)
+			continue
+		}
+		// Check that the head is no more than 2 slots behind the current time.
+		if beaconConfig.GetSlotTime(uint64(syncStatus.Data.HeadSlot)).Add(2 * time.Second * time.Duration(beaconConfig.SecondsPerSlot)).Before(time.Now()) {
+			// If the head is too far behind, wait and try again
+			time.Sleep(1 * time.Second)
+			continue
+		}
+
+		safe = true
+		break
+	}
+	if !safe {
+		return nil, fmt.Errorf("bn is not in sync after 30 seconds")
+	}
+
+	// Get the balances
+	return c.GetValidatorBalances(indices, opts)
+}
+
 // Get multiple validators' statuses
 func (c *StandardHttpClient) GetValidatorStatuses(pubkeys []types.ValidatorPubkey, opts *beacon.ValidatorStatusOptions) (map[types.ValidatorPubkey]beacon.ValidatorStatus, error) {
 

shared/services/beacon/client/types.go

@@ -32,9 +32,11 @@ type BLSToExecutionChangeRequest struct {
 // Response types
 type SyncStatusResponse struct {
 	Data struct {
-		IsSyncing    bool     `json:"is_syncing"`
 		HeadSlot     uinteger `json:"head_slot"`
 		SyncDistance uinteger `json:"sync_distance"`
+		IsSyncing    bool     `json:"is_syncing"`
+		IsOptimistic bool     `json:"is_optimistic"`
+		ELOffline    bool     `json:"el_offline"`
 	} `json:"data"`
 }
 type Eth2ConfigResponse struct {

shared/services/rewards/generator-impl-v9-v10.go

@@ -820,7 +820,7 @@ func (r *treeGeneratorImpl_v9_v10) getValidatorBalancesAtStartAndEnd() error {
 
 	r.log.Printlnf("%s Getting %d validator balances at start and end", r.logPrefix, len(indices))
 
-	validatorBalancesAtStart, err := r.bc.GetValidatorBalances(indices, &beacon.ValidatorStatusOptions{
+	validatorBalancesAtStart, err := r.bc.GetValidatorBalancesSafe(indices, &beacon.ValidatorStatusOptions{
 		Slot: &r.rewardsFile.ConsensusStartBlock,
 	})
 	if err != nil {
@@ -830,7 +830,7 @@ func (r *treeGeneratorImpl_v9_v10) getValidatorBalancesAtStartAndEnd() error {
 		r.validatorBalancesAtStart[r.validatorIndexMap[index].Address] = balance
 	}
 
-	validatorBalancesAtEnd, err := r.bc.GetValidatorBalances(indices, &beacon.ValidatorStatusOptions{
+	validatorBalancesAtEnd, err := r.bc.GetValidatorBalancesSafe(indices, &beacon.ValidatorStatusOptions{
 		Slot: &r.rewardsFile.ConsensusEndBlock,
 	})
 	if err != nil {
@@ -917,7 +917,7 @@ func (r *treeGeneratorImpl_v9_v10) getValidatorBalancesAtStartAndEnd() error {
 			r.log.Printlnf("%s On slot %d of %d (%.2f%%) - %.2f seconds per slot", r.logPrefix, i, total, float64(i)/float64(total)*100.0, secondsPerSlot)
 		}
 		i++
-		balances, err := r.bc.GetValidatorBalances(validatorIndices, &beacon.ValidatorStatusOptions{
+		balances, err := r.bc.GetValidatorBalancesSafe(validatorIndices, &beacon.ValidatorStatusOptions{
 			Slot: &slot,
 		})
 		if err != nil {
@@ -934,7 +934,7 @@ func (r *treeGeneratorImpl_v9_v10) getValidatorBalancesAtStartAndEnd() error {
 			r.log.Printlnf("%s On slot %d of %d (%.2f%%) - %.2f seconds per slot", r.logPrefix, i, total, float64(i)/float64(total)*100.0, secondsPerSlot)
 		}
 		i++
-		balances, err := r.bc.GetValidatorBalances(validatorIndices, &beacon.ValidatorStatusOptions{
+		balances, err := r.bc.GetValidatorBalancesSafe(validatorIndices, &beacon.ValidatorStatusOptions{
 			Slot: &slot,
 		})
 		if err != nil {

shared/services/rewards/test/beacon.go

@@ -422,6 +422,14 @@ func (bc *MockBeaconClient) SetCustomBalance(index string, balance *big.Int, slo
 	}{balance, slot})
 }
 
+// GetValidatorBalancesSafe returns the balances of the validators
+// The mock doesn't need to worry about thrashing the bn, since there is none.
+func (bc *MockBeaconClient) GetValidatorBalancesSafe(indices []string, opts *beacon.ValidatorStatusOptions) (map[string]*big.Int, error) {
+
+	// Get the balances
+	return bc.GetValidatorBalances(indices, opts)
+}
+
 func (bc *MockBeaconClient) GetValidatorBalances(indices []string, opts *beacon.ValidatorStatusOptions) (map[string]*big.Int, error) {
 	out := make(map[string]*big.Int)
 	for _, index := range indices {

shared/services/rewards/types.go

@@ -50,7 +50,7 @@ type RewardsBeaconClient interface {
 	GetBeaconBlock(slot string) (beacon.BeaconBlock, bool, error)
 	GetCommitteesForEpoch(epoch *uint64) (beacon.Committees, error)
 	GetAttestations(slot string) ([]beacon.AttestationInfo, bool, error)
-	GetValidatorBalances(indices []string, opts *beacon.ValidatorStatusOptions) (map[string]*big.Int, error)
+	GetValidatorBalancesSafe(indices []string, opts *beacon.ValidatorStatusOptions) (map[string]*big.Int, error)
 	GetEth2Config() (beacon.Eth2Config, error)
 	GetBeaconHead() (beacon.BeaconHead, error)
 }