diff --git a/cmd/misc/main.go b/cmd/misc/main.go index a0a05de291..9ebdccf2c1 100644 --- a/cmd/misc/main.go +++ b/cmd/misc/main.go @@ -226,22 +226,10 @@ func main() { logrus.Fatalf("error retrieving genesis validator state") } - data := &types.EpochData{ - SyncDuties: make(map[types.Slot]map[types.ValidatorIndex]bool), - AttestationDuties: make(map[types.Slot]map[types.ValidatorIndex][]types.Slot), - ValidatorAssignmentes: &types.EpochAssignments{ - ProposerAssignments: map[uint64]uint64{}, - AttestorAssignments: map[string]uint64{}, - SyncAssignments: make([]uint64, 0), - }, - Blocks: make(map[uint64]map[string]*types.Block), - FutureBlocks: make(map[uint64]map[string]*types.Block), - EpochParticipationStats: &types.ValidatorParticipation{}, - Finalized: false, - } + validatorsArr := make([]*types.Validator, 0, len(validators.Data)) for _, validator := range validators.Data { - data.Validators = append(data.Validators, &types.Validator{ + validatorsArr = append(validatorsArr, &types.Validator{ Index: uint64(validator.Index), PublicKey: utils.MustParseHex(validator.Validator.Pubkey), WithdrawalCredentials: utils.MustParseHex(validator.Validator.WithdrawalCredentials), @@ -256,15 +244,43 @@ func main() { }) } - tx := db.WriterDb.MustBegin() + batchSize := 10000 + for i := 0; i < len(validatorsArr); i += batchSize { + + data := &types.EpochData{ + SyncDuties: make(map[types.Slot]map[types.ValidatorIndex]bool), + AttestationDuties: make(map[types.Slot]map[types.ValidatorIndex][]types.Slot), + ValidatorAssignmentes: &types.EpochAssignments{ + ProposerAssignments: map[uint64]uint64{}, + AttestorAssignments: map[string]uint64{}, + SyncAssignments: make([]uint64, 0), + }, + Blocks: make(map[uint64]map[string]*types.Block), + FutureBlocks: make(map[uint64]map[string]*types.Block), + EpochParticipationStats: &types.ValidatorParticipation{}, + Finalized: false, + } - err = db.SaveValidators(data, tx, rpcClient, len(data.Validators)) - if err != nil { - logrus.Fatal(err) - } - err = tx.Commit() - if err != nil { - logrus.Fatal(err) + data.Validators = make([]*types.Validator, 0, batchSize) + + start := i + end := i + batchSize + if end >= len(validatorsArr) { + end = len(validatorsArr) - 1 + } + data.Validators = append(data.Validators, validatorsArr[start:end]...) + + logrus.Infof("saving validators %v-%v", data.Validators[0].Index, data.Validators[len(data.Validators)-1].Index) + tx := db.WriterDb.MustBegin() + + err = db.SaveValidators(data, tx, rpcClient, len(data.Validators)) + if err != nil { + logrus.Fatal(err) + } + err = tx.Commit() + if err != nil { + logrus.Fatal(err) + } } default: diff --git a/db/bigtable.go b/db/bigtable.go index a056d5e869..61ec2405de 100644 --- a/db/bigtable.go +++ b/db/bigtable.go @@ -6,6 +6,7 @@ import ( "eth2-exporter/types" "eth2-exporter/utils" "fmt" + "math" "os" "sort" "strconv" @@ -2014,7 +2015,9 @@ func (bigtable *Bigtable) DeleteEpoch(epoch uint64) error { } func (bigtable *Bigtable) getValidatorsEpochRanges(validatorIndices []uint64, prefix string, startEpoch uint64, endEpoch uint64) gcp_bigtable.RowRangeList { - + if endEpoch > math.MaxInt64 { + endEpoch = 0 + } if endEpoch < startEpoch { // handle overflows startEpoch = 0 } @@ -2035,6 +2038,9 @@ func (bigtable *Bigtable) getValidatorsEpochRanges(validatorIndices []uint64, pr func (bigtable *Bigtable) getValidatorsEpochSlotRanges(validatorIndices []uint64, prefix string, startEpoch uint64, endEpoch uint64) gcp_bigtable.RowRangeList { + if endEpoch > math.MaxInt64 { + endEpoch = 0 + } if endEpoch < startEpoch { // handle overflows startEpoch = 0 } @@ -2053,7 +2059,9 @@ func (bigtable *Bigtable) getValidatorsEpochSlotRanges(validatorIndices []uint64 } func (bigtable *Bigtable) getValidatorSlotRanges(validatorIndices []uint64, prefix string, startSlot uint64, endSlot uint64) gcp_bigtable.RowRangeList { - + if endSlot > math.MaxInt64 { + endSlot = 0 + } if endSlot < startSlot { // handle overflows startSlot = 0 } diff --git a/db/db.go b/db/db.go index dc3db7b038..ac02dd576f 100644 --- a/db/db.go +++ b/db/db.go @@ -1081,7 +1081,7 @@ func SaveValidators(data *types.EpochData, tx *sqlx.Tx, client rpc.Client, activ err := tx.Select(¤tState, "SELECT validatorindex, withdrawableepoch, withdrawalcredentials, slashed, activationeligibilityepoch, activationepoch, exitepoch, status FROM validators;") if err != nil { - return err + return fmt.Errorf("error retrieving current validator state set: %v", err) } lastAttestationSlots, err := BigtableClient.GetLastAttestationSlots([]uint64{}) @@ -1278,7 +1278,7 @@ func SaveValidators(data *types.EpochData, tx *sqlx.Tx, client rpc.Client, activ ActivationEpoch uint64 }{} - err = tx.Select(&newValidators, "SELECT validatorindex, activationepoch FROM validators WHERE balanceactivation IS NULL ORDER BY activationepoch LIMIT ?", activationBalanceBatchSize) + err = tx.Select(&newValidators, "SELECT validatorindex, activationepoch FROM validators WHERE balanceactivation IS NULL ORDER BY activationepoch LIMIT $1", activationBalanceBatchSize) if err != nil { return err } diff --git a/handlers/validator.go b/handlers/validator.go index 094803a7c7..87db380841 100644 --- a/handlers/validator.go +++ b/handlers/validator.go @@ -78,6 +78,12 @@ func Validator(w http.ResponseWriter, r *http.Request) { latestProposedSlot := services.LatestProposedSlot() lastFinalizedEpoch := services.LatestFinalizedEpoch() + if latestProposedSlot == 0 { + latestEpoch = 1 + latestProposedSlot = 1 + lastFinalizedEpoch = 1 + } + validatorPageData := types.ValidatorPageData{} validatorPageData.CappellaHasHappened = latestEpoch >= (utils.Config.Chain.Config.CappellaForkEpoch) diff --git a/services/services.go b/services/services.go index 998f5444ad..95e8a23ced 100644 --- a/services/services.go +++ b/services/services.go @@ -691,6 +691,18 @@ func getIndexPageData() (*types.IndexPageData, error) { return nil, fmt.Errorf("error retrieving eth1 deposits: %v", err) } + if deposit.Total == 0 { // see if there are any genesis validators + err = db.ReaderDb.Get(&deposit.Total, "SELECT COALESCE(MAX(validatorindex), 0) FROM validators") + if err != nil { + return nil, fmt.Errorf("error retrieving max validator index: %v", err) + } + + if deposit.Total > 0 { + deposit.Total = (deposit.Total + 1) * 32 + deposit.BlockTs = time.Now() + } + } + threshold, err := db.GetDepositThresholdTime() if err != nil { logger.WithError(err).Error("error could not calculate threshold time")