(BIDS-2472) fix out of memory issue

cmd/misc/main.go

@@ -226,22 +226,10 @@ func main() {
 			logrus.Fatalf("error retrieving genesis validator state")
 		}
 
-		data := &types.EpochData{
-			SyncDuties:        make(map[types.Slot]map[types.ValidatorIndex]bool),
-			AttestationDuties: make(map[types.Slot]map[types.ValidatorIndex][]types.Slot),
-			ValidatorAssignmentes: &types.EpochAssignments{
-				ProposerAssignments: map[uint64]uint64{},
-				AttestorAssignments: map[string]uint64{},
-				SyncAssignments:     make([]uint64, 0),
-			},
-			Blocks:                  make(map[uint64]map[string]*types.Block),
-			FutureBlocks:            make(map[uint64]map[string]*types.Block),
-			EpochParticipationStats: &types.ValidatorParticipation{},
-			Finalized:               false,
-		}
+		validatorsArr := make([]*types.Validator, 0, len(validators.Data))
 
 		for _, validator := range validators.Data {
-			data.Validators = append(data.Validators, &types.Validator{
+			validatorsArr = append(validatorsArr, &types.Validator{
 				Index:                      uint64(validator.Index),
 				PublicKey:                  utils.MustParseHex(validator.Validator.Pubkey),
 				WithdrawalCredentials:      utils.MustParseHex(validator.Validator.WithdrawalCredentials),
@@ -256,15 +244,43 @@ func main() {
 			})
 		}
 
-		tx := db.WriterDb.MustBegin()
+		batchSize := 10000
+		for i := 0; i < len(validatorsArr); i += batchSize {
+
+			data := &types.EpochData{
+				SyncDuties:        make(map[types.Slot]map[types.ValidatorIndex]bool),
+				AttestationDuties: make(map[types.Slot]map[types.ValidatorIndex][]types.Slot),
+				ValidatorAssignmentes: &types.EpochAssignments{
+					ProposerAssignments: map[uint64]uint64{},
+					AttestorAssignments: map[string]uint64{},
+					SyncAssignments:     make([]uint64, 0),
+				},
+				Blocks:                  make(map[uint64]map[string]*types.Block),
+				FutureBlocks:            make(map[uint64]map[string]*types.Block),
+				EpochParticipationStats: &types.ValidatorParticipation{},
+				Finalized:               false,
+			}
 
-		err = db.SaveValidators(data, tx, rpcClient, len(data.Validators))
-		if err != nil {
-			logrus.Fatal(err)
-		}
-		err = tx.Commit()
-		if err != nil {
-			logrus.Fatal(err)
+			data.Validators = make([]*types.Validator, 0, batchSize)
+
+			start := i
+			end := i + batchSize
+			if end >= len(validatorsArr) {
+				end = len(validatorsArr) - 1
+			}
+			data.Validators = append(data.Validators, validatorsArr[start:end]...)
+
+			logrus.Infof("saving validators %v-%v", data.Validators[0].Index, data.Validators[len(data.Validators)-1].Index)
+			tx := db.WriterDb.MustBegin()
+
+			err = db.SaveValidators(data, tx, rpcClient, len(data.Validators))
+			if err != nil {
+				logrus.Fatal(err)
+			}
+			err = tx.Commit()
+			if err != nil {
+				logrus.Fatal(err)
+			}
 		}
 
 	default:

db/bigtable.go

@@ -6,6 +6,7 @@ import (
 	"eth2-exporter/types"
 	"eth2-exporter/utils"
 	"fmt"
+	"math"
 	"os"
 	"sort"
 	"strconv"
@@ -2014,7 +2015,9 @@ func (bigtable *Bigtable) DeleteEpoch(epoch uint64) error {
 }
 
 func (bigtable *Bigtable) getValidatorsEpochRanges(validatorIndices []uint64, prefix string, startEpoch uint64, endEpoch uint64) gcp_bigtable.RowRangeList {
-
+	if endEpoch > math.MaxInt64 {
+		endEpoch = 0
+	}
 	if endEpoch < startEpoch { // handle overflows
 		startEpoch = 0
 	}
@@ -2035,6 +2038,9 @@ func (bigtable *Bigtable) getValidatorsEpochRanges(validatorIndices []uint64, pr
 
 func (bigtable *Bigtable) getValidatorsEpochSlotRanges(validatorIndices []uint64, prefix string, startEpoch uint64, endEpoch uint64) gcp_bigtable.RowRangeList {
 
+	if endEpoch > math.MaxInt64 {
+		endEpoch = 0
+	}
 	if endEpoch < startEpoch { // handle overflows
 		startEpoch = 0
 	}
@@ -2053,7 +2059,9 @@ func (bigtable *Bigtable) getValidatorsEpochSlotRanges(validatorIndices []uint64
 }
 
 func (bigtable *Bigtable) getValidatorSlotRanges(validatorIndices []uint64, prefix string, startSlot uint64, endSlot uint64) gcp_bigtable.RowRangeList {
-
+	if endSlot > math.MaxInt64 {
+		endSlot = 0
+	}
 	if endSlot < startSlot { // handle overflows
 		startSlot = 0
 	}

db/db.go

@@ -1081,7 +1081,7 @@ func SaveValidators(data *types.EpochData, tx *sqlx.Tx, client rpc.Client, activ
 	err := tx.Select(&currentState, "SELECT validatorindex, withdrawableepoch, withdrawalcredentials, slashed, activationeligibilityepoch, activationepoch, exitepoch, status FROM validators;")
 
 	if err != nil {
-		return err
+		return fmt.Errorf("error retrieving current validator state set: %v", err)
 	}
 
 	lastAttestationSlots, err := BigtableClient.GetLastAttestationSlots([]uint64{})
@@ -1278,7 +1278,7 @@ func SaveValidators(data *types.EpochData, tx *sqlx.Tx, client rpc.Client, activ
 		ActivationEpoch uint64
 	}{}
 
-	err = tx.Select(&newValidators, "SELECT validatorindex, activationepoch FROM validators WHERE balanceactivation IS NULL ORDER BY activationepoch LIMIT ?", activationBalanceBatchSize)
+	err = tx.Select(&newValidators, "SELECT validatorindex, activationepoch FROM validators WHERE balanceactivation IS NULL ORDER BY activationepoch LIMIT $1", activationBalanceBatchSize)
 	if err != nil {
 		return err
 	}

handlers/validator.go

@@ -78,6 +78,12 @@ func Validator(w http.ResponseWriter, r *http.Request) {
 	latestProposedSlot := services.LatestProposedSlot()
 	lastFinalizedEpoch := services.LatestFinalizedEpoch()
 
+	if latestProposedSlot == 0 {
+		latestEpoch = 1
+		latestProposedSlot = 1
+		lastFinalizedEpoch = 1
+	}
+
 	validatorPageData := types.ValidatorPageData{}
 
 	validatorPageData.CappellaHasHappened = latestEpoch >= (utils.Config.Chain.Config.CappellaForkEpoch)

services/services.go

@@ -691,6 +691,18 @@ func getIndexPageData() (*types.IndexPageData, error) {
 			return nil, fmt.Errorf("error retrieving eth1 deposits: %v", err)
 		}
 
+		if deposit.Total == 0 { // see if there are any genesis validators
+			err = db.ReaderDb.Get(&deposit.Total, "SELECT COALESCE(MAX(validatorindex), 0) FROM validators")
+			if err != nil {
+				return nil, fmt.Errorf("error retrieving max validator index: %v", err)
+			}
+
+			if deposit.Total > 0 {
+				deposit.Total = (deposit.Total + 1) * 32
+				deposit.BlockTs = time.Now()
+			}
+		}
+
 		threshold, err := db.GetDepositThresholdTime()
 		if err != nil {
 			logger.WithError(err).Error("error could not calculate threshold time")