From 0d382f054d82c9d86d44ec8c0cef781a88c88541 Mon Sep 17 00:00:00 2001 From: peter <1674920+peterbitfly@users.noreply.github.com> Date: Mon, 21 Aug 2023 10:44:15 +0200 Subject: [PATCH 001/106] (BIDS-2379) implement new bigtable cl schema --- db/bigtable.go | 977 ++++++++++++++++++------------------- db/db.go | 7 + db/statistics.go | 74 ++- handlers/validator.go | 4 +- services/charts_updater.go | 14 +- services/notifications.go | 8 +- services/services.go | 8 +- utils/utils.go | 8 + 8 files changed, 541 insertions(+), 559 deletions(-) diff --git a/db/bigtable.go b/db/bigtable.go index e1f6703b8a..b1420e8ed6 100644 --- a/db/bigtable.go +++ b/db/bigtable.go @@ -15,7 +15,7 @@ import ( gcp_bigtable "cloud.google.com/go/bigtable" "github.com/go-redis/redis/v8" itypes "github.com/gobitfly/eth-rewards/types" - utilMath "github.com/protolambda/zrnt/eth2/util/math" + "github.com/sirupsen/logrus" "golang.org/x/sync/errgroup" "google.golang.org/api/option" "google.golang.org/protobuf/proto" @@ -43,14 +43,20 @@ const ( type Bigtable struct { client *gcp_bigtable.Client - tableBeaconchain *gcp_bigtable.Table + tableBeaconchain *gcp_bigtable.Table + tableValidators *gcp_bigtable.Table + tableValidatorBalances *gcp_bigtable.Table + tableValidatorAttestations *gcp_bigtable.Table + tableValidatorProposals *gcp_bigtable.Table + tableValidatorSyncCommittees *gcp_bigtable.Table + tableValidatorIncomeDetails *gcp_bigtable.Table tableData *gcp_bigtable.Table tableBlocks *gcp_bigtable.Table tableMetadataUpdates *gcp_bigtable.Table tableMetadata *gcp_bigtable.Table - tableMachineMetrics *gcp_bigtable.Table - tableValidators *gcp_bigtable.Table + + tableMachineMetrics *gcp_bigtable.Table redisCache *redis.Client @@ -82,17 +88,22 @@ func InitBigtable(project, instance, chainId, redisAddress string) (*Bigtable, e } bt := &Bigtable{ - client: btClient, - tableData: btClient.Open("data"), - tableBlocks: btClient.Open("blocks"), - tableMetadataUpdates: btClient.Open("metadata_updates"), - tableMetadata: btClient.Open("metadata"), - tableBeaconchain: btClient.Open("beaconchain"), - tableMachineMetrics: btClient.Open("machine_metrics"), - tableValidators: btClient.Open("beaconchain_validators"), - chainId: chainId, - redisCache: rdc, - lastAttestationCacheMux: &sync.Mutex{}, + client: btClient, + tableData: btClient.Open("data"), + tableBlocks: btClient.Open("blocks"), + tableMetadataUpdates: btClient.Open("metadata_updates"), + tableMetadata: btClient.Open("metadata"), + tableBeaconchain: btClient.Open("beaconchain"), + tableMachineMetrics: btClient.Open("machine_metrics"), + tableValidators: btClient.Open("beaconchain_validators"), + tableValidatorBalances: btClient.Open("beaconchain_validator_balances"), + tableValidatorAttestations: btClient.Open("beaconchain_validator_attestations"), + tableValidatorProposals: btClient.Open("beaconchain_validator_proposals"), + tableValidatorSyncCommittees: btClient.Open("beaconchain_validator_sync"), + tableValidatorIncomeDetails: btClient.Open("beaconchain_validator_income"), + chainId: chainId, + redisCache: rdc, + lastAttestationCacheMux: &sync.Mutex{}, } BigtableClient = bt @@ -394,7 +405,8 @@ func (bigtable *Bigtable) SaveValidatorBalances(epoch uint64, validators []*type start := time.Now() ts := gcp_bigtable.Timestamp(0) - mut := gcp_bigtable.NewMutation() + muts := make([]*gcp_bigtable.Mutation, 0, 100000) + keys := make([]string, 0, 100000) for i, validator := range validators { balanceEncoded := make([]byte, 8) @@ -404,21 +416,38 @@ func (bigtable *Bigtable) SaveValidatorBalances(epoch uint64, validators []*type binary.LittleEndian.PutUint64(effectiveBalanceEncoded, validator.EffectiveBalance) combined := append(balanceEncoded, effectiveBalanceEncoded...) - mut.Set(VALIDATOR_BALANCES_FAMILY, fmt.Sprintf("%d", validator.Index), ts, combined) + mut := &gcp_bigtable.Mutation{} + mut.Set(VALIDATOR_BALANCES_FAMILY, "b", ts, combined) + key := fmt.Sprintf("%s:%s:%s", bigtable.chainId, bigtable.validatorIndexToKey(validator.Index), reversedPaddedEpoch(epoch)) + + muts = append(muts, mut) + keys = append(keys, key) if i%100000 == 0 { - err := bigtable.tableBeaconchain.Apply(ctx, fmt.Sprintf("%s:e:b:%s", bigtable.chainId, reversedPaddedEpoch(epoch)), mut) + errs, err := bigtable.tableValidatorBalances.ApplyBulk(ctx, keys, muts) if err != nil { return err } - mut = gcp_bigtable.NewMutation() + + for _, err := range errs { + return err + } + muts = make([]*gcp_bigtable.Mutation, 0, 100000) + keys = make([]string, 0, 100000) } } - err := bigtable.tableBeaconchain.Apply(ctx, fmt.Sprintf("%s:e:b:%s", bigtable.chainId, reversedPaddedEpoch(epoch)), mut) - if err != nil { - return err + if len(muts) > 0 { + errs, err := bigtable.tableValidatorBalances.ApplyBulk(ctx, keys, muts) + + if err != nil { + return err + } + + for _, err := range errs { + return err + } } logger.Infof("exported validator balances to bigtable in %v", time.Since(start)) @@ -449,15 +478,25 @@ func (bigtable *Bigtable) SaveAttestationAssignments(epoch uint64, assignments m } for slot, validators := range validatorsPerSlot { - mut := gcp_bigtable.NewMutation() + muts := make([]*gcp_bigtable.Mutation, 0, 100000) + keys := make([]string, 0, 100000) + for _, validator := range validators { - mut.Set(ATTESTATIONS_FAMILY, fmt.Sprintf("%d", validator), ts, []byte{}) + mut := gcp_bigtable.NewMutation() + key := fmt.Sprintf("%s:%s:%s", bigtable.chainId, bigtable.validatorIndexToKey(validator), reversedPaddedEpoch(epoch)) + mut.Set(ATTESTATIONS_FAMILY, fmt.Sprintf("%d", slot), ts, []byte{}) + + muts = append(muts, mut) + keys = append(keys, key) } - err := bigtable.tableBeaconchain.Apply(ctx, fmt.Sprintf("%s:e:%s:s:%s", bigtable.chainId, reversedPaddedEpoch(epoch), reversedPaddedSlot(slot)), mut) + errs, err := bigtable.tableValidatorAttestations.ApplyBulk(ctx, keys, muts) if err != nil { return err } + for _, err := range errs { + return err + } } logger.Infof("exported attestation assignments to bigtable in %v", time.Since(start)) @@ -472,14 +511,27 @@ func (bigtable *Bigtable) SaveProposalAssignments(epoch uint64, assignments map[ start := time.Now() ts := gcp_bigtable.Timestamp(0) + muts := make([]*gcp_bigtable.Mutation, 0, 100000) + keys := make([]string, 0, 100000) + for slot, validator := range assignments { mut := gcp_bigtable.NewMutation() - mut.Set(PROPOSALS_FAMILY, fmt.Sprintf("%d", validator), ts, []byte{}) - err := bigtable.tableBeaconchain.Apply(ctx, fmt.Sprintf("%s:e:%s:s:%s", bigtable.chainId, reversedPaddedEpoch(epoch), reversedPaddedSlot(slot)), mut) + mut.Set(PROPOSALS_FAMILY, "p", ts, []byte{}) - if err != nil { - return err - } + key := fmt.Sprintf("%s:%s:%s:%s", bigtable.chainId, bigtable.validatorIndexToKey(validator), reversedPaddedEpoch(epoch), reversedPaddedSlot(slot)) + + muts = append(muts, mut) + keys = append(keys, key) + } + + errs, err := bigtable.tableValidatorProposals.ApplyBulk(ctx, keys, muts) + + if err != nil { + return err + } + + for _, err := range errs { + return err } logger.Infof("exported proposal assignments to bigtable in %v", time.Since(start)) @@ -498,18 +550,21 @@ func (bigtable *Bigtable) SaveSyncCommitteesAssignments(startSlot, endSlot uint6 var keys []string for i := startSlot; i <= endSlot; i++ { - mut := gcp_bigtable.NewMutation() for _, validator := range validators { - mut.Set(SYNC_COMMITTEES_FAMILY, fmt.Sprintf("%d", validator), ts, []byte{}) + mut := gcp_bigtable.NewMutation() + mut.Set(SYNC_COMMITTEES_FAMILY, "s", ts, []byte{}) + + key := fmt.Sprintf("%s:%s:%s:%s", bigtable.chainId, bigtable.validatorIndexToKey(validator), reversedPaddedEpoch(i/utils.Config.Chain.Config.SlotsPerEpoch), reversedPaddedSlot(i)) + + muts = append(muts, mut) + keys = append(keys, key) } - muts = append(muts, mut) - keys = append(keys, fmt.Sprintf("%s:e:%s:s:%s", bigtable.chainId, reversedPaddedEpoch(i/utils.Config.Chain.Config.SlotsPerEpoch), reversedPaddedSlot(i))) } logger.Infof("saving %v mutations for sync duties", len(muts)) - errs, err := bigtable.tableBeaconchain.ApplyBulk(ctx, keys, muts) + errs, err := bigtable.tableValidatorSyncCommittees.ApplyBulk(ctx, keys, muts) if err != nil { return err @@ -560,7 +615,7 @@ func (bigtable *Bigtable) SaveAttestations(blocks map[uint64]map[string]*types.B for _, slot := range slots { for _, b := range blocks[slot] { - logger.Infof("processing slot %v", slot) + // logger.Infof("processing slot %v", slot) for _, a := range b.Attestations { for _, validator := range a.Attesters { inclusionSlot := slot @@ -578,27 +633,36 @@ func (bigtable *Bigtable) SaveAttestations(blocks map[uint64]map[string]*types.B } for attestedSlot, inclusions := range attestationsBySlot { - mutInclusionSlot := gcp_bigtable.NewMutation() + mutsInclusionSlot := make([]*gcp_bigtable.Mutation, 0, 100000) + keysInclusionSlot := make([]string, 0, 100000) mutLastAttestationSlot := gcp_bigtable.NewMutation() mutLastAttestationSlotSet := false + epoch := attestedSlot / utils.Config.Chain.Config.SlotsPerEpoch bigtable.lastAttestationCacheMux.Lock() for validator, inclusionSlot := range inclusions { - mutInclusionSlot.Set(ATTESTATIONS_FAMILY, fmt.Sprintf("%d", validator), gcp_bigtable.Timestamp((max_block_number-inclusionSlot)*1000), []byte{}) + mutInclusionSlot := gcp_bigtable.NewMutation() + mutInclusionSlot.Set(ATTESTATIONS_FAMILY, fmt.Sprintf("%d", attestedSlot), gcp_bigtable.Timestamp((max_block_number-inclusionSlot)*1000), []byte{}) + key := fmt.Sprintf("%s:%s:%s", bigtable.chainId, bigtable.validatorIndexToKey(validator), reversedPaddedEpoch(epoch)) + + mutsInclusionSlot = append(mutsInclusionSlot, mutInclusionSlot) + keysInclusionSlot = append(keysInclusionSlot, key) if attestedSlot > bigtable.lastAttestationCache[validator] { mutLastAttestationSlot.Set(ATTESTATIONS_FAMILY, fmt.Sprintf("%d", validator), gcp_bigtable.Timestamp((attestedSlot)*1000), []byte{}) bigtable.lastAttestationCache[validator] = attestedSlot mutLastAttestationSlotSet = true } - } bigtable.lastAttestationCacheMux.Unlock() - err := bigtable.tableBeaconchain.Apply(ctx, fmt.Sprintf("%s:e:%s:s:%s", bigtable.chainId, reversedPaddedEpoch(attestedSlot/utils.Config.Chain.Config.SlotsPerEpoch), reversedPaddedSlot(attestedSlot)), mutInclusionSlot) + errs, err := bigtable.tableValidatorAttestations.ApplyBulk(ctx, keysInclusionSlot, mutsInclusionSlot) if err != nil { return err } + for _, err := range errs { + return err + } if mutLastAttestationSlotSet { err = bigtable.tableValidators.Apply(ctx, fmt.Sprintf("%s:lastAttestationSlot", bigtable.chainId), mutLastAttestationSlot) @@ -642,6 +706,9 @@ func (bigtable *Bigtable) SaveProposals(blocks map[uint64]map[string]*types.Bloc return slots[i] < slots[j] }) + muts := make([]*gcp_bigtable.Mutation, 0, 100000) + keys := make([]string, 0, 100000) + for _, slot := range slots { for _, b := range blocks[slot] { @@ -649,13 +716,22 @@ func (bigtable *Bigtable) SaveProposals(blocks map[uint64]map[string]*types.Bloc continue } mut := gcp_bigtable.NewMutation() - mut.Set(PROPOSALS_FAMILY, fmt.Sprintf("%d", b.Proposer), gcp_bigtable.Timestamp((max_block_number-b.Slot)*1000), []byte{}) - err := bigtable.tableBeaconchain.Apply(ctx, fmt.Sprintf("%s:e:%s:s:%s", bigtable.chainId, reversedPaddedEpoch(b.Slot/utils.Config.Chain.Config.SlotsPerEpoch), reversedPaddedSlot(b.Slot)), mut) - if err != nil { - return err - } + mut.Set(PROPOSALS_FAMILY, "b", gcp_bigtable.Timestamp((max_block_number-b.Slot)*1000), []byte{}) + key := fmt.Sprintf("%s:%s:%s:%s", bigtable.chainId, bigtable.validatorIndexToKey(b.Proposer), reversedPaddedEpoch(b.Slot/utils.Config.Chain.Config.SlotsPerEpoch), reversedPaddedSlot(b.Slot)) + + muts = append(muts, mut) + keys = append(keys, key) } } + errs, err := bigtable.tableValidatorProposals.ApplyBulk(ctx, keys, muts) + + if err != nil { + return err + } + + for _, err := range errs { + return err + } logger.Infof("exported proposals to bigtable in %v", time.Since(start)) return nil } @@ -701,86 +777,67 @@ func (bigtable *Bigtable) SaveSyncComitteeDuties(blocks map[uint64]map[string]*t logger.Infof("no sync duties to export") return nil } + + muts := make([]*gcp_bigtable.Mutation, 0, 100000) + keys := make([]string, 0, 100000) + for slot, validators := range dutiesBySlot { - mut := gcp_bigtable.NewMutation() for validator, participated := range validators { + mut := gcp_bigtable.NewMutation() if participated { - mut.Set(SYNC_COMMITTEES_FAMILY, fmt.Sprintf("%d", validator), gcp_bigtable.Timestamp((max_block_number-slot)*1000), []byte{}) + mut.Set(SYNC_COMMITTEES_FAMILY, "s", gcp_bigtable.Timestamp((max_block_number-slot)*1000), []byte{}) } else { - mut.Set(SYNC_COMMITTEES_FAMILY, fmt.Sprintf("%d", validator), gcp_bigtable.Timestamp(0), []byte{}) + mut.Set(SYNC_COMMITTEES_FAMILY, "s", gcp_bigtable.Timestamp(0), []byte{}) } + key := fmt.Sprintf("%s:%s:%s:%s", bigtable.chainId, bigtable.validatorIndexToKey(validator), reversedPaddedEpoch(slot/utils.Config.Chain.Config.SlotsPerEpoch), reversedPaddedSlot(slot)) + muts = append(muts, mut) + keys = append(keys, key) } - err := bigtable.tableBeaconchain.Apply(ctx, fmt.Sprintf("%s:e:%s:s:%s", bigtable.chainId, reversedPaddedEpoch(slot/utils.Config.Chain.Config.SlotsPerEpoch), reversedPaddedSlot(slot)), mut) - if err != nil { - return err - } } + + errs, err := bigtable.tableValidatorSyncCommittees.ApplyBulk(ctx, keys, muts) + + if err != nil { + return err + } + + for _, err := range errs { + return err + } + logger.Infof("exported sync committee duties to bigtable in %v", time.Since(start)) return nil } -func (bigtable *Bigtable) GetValidatorBalanceHistory(validators []uint64, startEpoch uint64, endEpoch uint64) (map[uint64][]*types.ValidatorBalance, error) { +func (bigtable *Bigtable) GetSingleValidatorBalanceHistory(validator uint64, startEpoch uint64, endEpoch uint64) ([]*types.ValidatorBalance, error) { - valLen := len(validators) - getAllThreshold := 1000 - validatorMap := make(map[uint64]bool, valLen) - for _, validatorIndex := range validators { - validatorMap[validatorIndex] = true + if endEpoch < startEpoch { // handle overflows + startEpoch = 0 } ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(time.Second*30)) defer cancel() - ranges := bigtable.getEpochRanges(startEpoch, endEpoch) - res := make(map[uint64][]*types.ValidatorBalance, valLen) - - columnFilters := []gcp_bigtable.Filter{} - if valLen < getAllThreshold { - columnFilters = make([]gcp_bigtable.Filter, 0, valLen) - for _, validator := range validators { - columnFilters = append(columnFilters, gcp_bigtable.ColumnFilter(fmt.Sprintf("%d", validator))) - } - } - - filter := gcp_bigtable.ChainFilters( - gcp_bigtable.FamilyFilter(VALIDATOR_BALANCES_FAMILY), - gcp_bigtable.InterleaveFilters(columnFilters...), - ) - - if len(columnFilters) == 1 { // special case to retrieve data for one validators - filter = gcp_bigtable.ChainFilters( - gcp_bigtable.FamilyFilter(VALIDATOR_BALANCES_FAMILY), - columnFilters[0], - ) - } - if len(columnFilters) == 0 { // special case to retrieve data for all validators - filter = gcp_bigtable.FamilyFilter(VALIDATOR_BALANCES_FAMILY) - } + ranges := bigtable.getValidatorEpochRanges(validator, startEpoch, endEpoch) + res := make([]*types.ValidatorBalance, 0, endEpoch-startEpoch) + ro := gcp_bigtable.LimitRows(int64(endEpoch - startEpoch + 1)) handleRow := func(r gcp_bigtable.Row) bool { keySplit := strings.Split(r.Key(), ":") - epoch, err := strconv.ParseUint(keySplit[3], 10, 64) + epoch, err := strconv.ParseUint(keySplit[2], 10, 64) if err != nil { logger.Errorf("error parsing epoch from row key %v: %v", r.Key(), err) return false } + validator, err := bigtable.validatorKeyToIndex(keySplit[1]) + if err != nil { + logger.Errorf("error parsing validator index from row key %v: %v", r.Key(), err) + return false + } for _, ri := range r[VALIDATOR_BALANCES_FAMILY] { - validator, err := strconv.ParseUint(strings.TrimPrefix(ri.Column, VALIDATOR_BALANCES_FAMILY+":"), 10, 64) - if err != nil { - logger.Errorf("error parsing validator from column key %v: %v", ri.Column, err) - return false - } - - // If we requested more than getAllThreshold validators we will - // get data for all validators and need to filter out all - // unwanted ones - if valLen >= getAllThreshold && !validatorMap[validator] { - continue - } - balances := ri.Value balanceBytes := balances[0:8] @@ -788,11 +845,7 @@ func (bigtable *Bigtable) GetValidatorBalanceHistory(validators []uint64, startE balance := binary.LittleEndian.Uint64(balanceBytes) effectiveBalance := binary.LittleEndian.Uint64(effectiveBalanceBytes) - if res[validator] == nil { - res[validator] = make([]*types.ValidatorBalance, 0) - } - - res[validator] = append(res[validator], &types.ValidatorBalance{ + res = append(res, &types.ValidatorBalance{ Epoch: max_epoch - epoch, Balance: balance, EffectiveBalance: effectiveBalance, @@ -803,7 +856,7 @@ func (bigtable *Bigtable) GetValidatorBalanceHistory(validators []uint64, startE return true } - err := bigtable.tableBeaconchain.ReadRows(ctx, ranges, handleRow, gcp_bigtable.RowFilter(filter)) + err := bigtable.tableValidatorBalances.ReadRows(ctx, ranges, handleRow, ro) if err != nil { return nil, err } @@ -811,8 +864,101 @@ func (bigtable *Bigtable) GetValidatorBalanceHistory(validators []uint64, startE return res, nil } +func (bigtable *Bigtable) GetValidatorBalanceHistory(validators []uint64, startEpoch uint64, endEpoch uint64) (map[uint64][]*types.ValidatorBalance, error) { + + if len(validators) == 0 { + return nil, fmt.Errorf("passing empty validator array is unsupported") + } + + batchSize := 1000 + concurrency := 10 + + ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(time.Minute*5)) + defer cancel() + + res := make(map[uint64][]*types.ValidatorBalance, len(validators)) + resMux := &sync.Mutex{} + + g, gCtx := errgroup.WithContext(ctx) + g.SetLimit(concurrency) + + for i := 0; i < len(validators); i += batchSize { + + upperBound := i + batchSize + if len(validators) < upperBound { + upperBound = len(validators) + } + vals := validators[i:upperBound] + + g.Go(func() error { + select { + case <-gCtx.Done(): + return gCtx.Err() + default: + } + ranges := bigtable.getValidatorsEpochRanges(vals, startEpoch, endEpoch) + ro := gcp_bigtable.LimitRows(int64(endEpoch-startEpoch+1) * int64(len(vals))) + + handleRow := func(r gcp_bigtable.Row) bool { + keySplit := strings.Split(r.Key(), ":") + + epoch, err := strconv.ParseUint(keySplit[2], 10, 64) + if err != nil { + logger.Errorf("error parsing epoch from row key %v: %v", r.Key(), err) + return false + } + + validator, err := bigtable.validatorKeyToIndex(keySplit[1]) + if err != nil { + logger.Errorf("error parsing validator index from row key %v: %v", r.Key(), err) + return false + } + resMux.Lock() + if res[validator] == nil { + res[validator] = make([]*types.ValidatorBalance, 0) + } + resMux.Unlock() + + for _, ri := range r[VALIDATOR_BALANCES_FAMILY] { + + balances := ri.Value + + balanceBytes := balances[0:8] + effectiveBalanceBytes := balances[8:16] + balance := binary.LittleEndian.Uint64(balanceBytes) + effectiveBalance := binary.LittleEndian.Uint64(effectiveBalanceBytes) + + resMux.Lock() + res[validator] = append(res[validator], &types.ValidatorBalance{ + Epoch: max_epoch - epoch, + Balance: balance, + EffectiveBalance: effectiveBalance, + Index: validator, + PublicKey: []byte{}, + }) + resMux.Unlock() + } + return true + } + + err := bigtable.tableValidatorBalances.ReadRows(gCtx, ranges, handleRow, ro) + if err != nil { + return err + } + + logrus.Infof("retrieved data for validators %v - %v", vals[0], vals[len(vals)-1]) + return nil + }) + } + + if err := g.Wait(); err != nil { + return nil, err + } + + return res, nil +} + func (bigtable *Bigtable) GetValidatorAttestationHistory(validators []uint64, startEpoch uint64, endEpoch uint64) (map[uint64][]*types.ValidatorAttestation, error) { - valLen := len(validators) ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(time.Minute*5)) defer cancel() @@ -827,46 +973,26 @@ func (bigtable *Bigtable) GetValidatorAttestationHistory(validators []uint64, st return nil, err } - ranges := bigtable.getSlotRanges(startEpoch, endEpoch) + ranges := bigtable.getValidatorsEpochRanges(validators, startEpoch, endEpoch) res := make(map[uint64][]*types.ValidatorAttestation, len(validators)) - columnFilters := []gcp_bigtable.Filter{} - if valLen < 1000 { - columnFilters = make([]gcp_bigtable.Filter, 0, len(validators)) - for _, validator := range validators { - columnFilters = append(columnFilters, gcp_bigtable.ColumnFilter(fmt.Sprintf("%d", validator))) - } - } - - filter := gcp_bigtable.ChainFilters( - gcp_bigtable.FamilyFilter(ATTESTATIONS_FAMILY), - gcp_bigtable.InterleaveFilters(columnFilters...), - gcp_bigtable.LatestNFilter(1), - ) + filter := gcp_bigtable.LatestNFilter(1) - if len(columnFilters) == 1 { // special case to retrieve data for one validators - filter = gcp_bigtable.ChainFilters( - gcp_bigtable.FamilyFilter(ATTESTATIONS_FAMILY), - columnFilters[0], - gcp_bigtable.LatestNFilter(1), - ) - } - if len(columnFilters) == 0 { // special case to retrieve data for all validators - filter = gcp_bigtable.ChainFilters( - gcp_bigtable.FamilyFilter(ATTESTATIONS_FAMILY), - gcp_bigtable.LatestNFilter(1), - ) - } - err = bigtable.tableBeaconchain.ReadRows(ctx, ranges, func(r gcp_bigtable.Row) bool { + err = bigtable.tableValidatorAttestations.ReadRows(ctx, ranges, func(r gcp_bigtable.Row) bool { keySplit := strings.Split(r.Key(), ":") - attesterSlot, err := strconv.ParseUint(keySplit[4], 10, 64) + validator, err := bigtable.validatorKeyToIndex(keySplit[1]) if err != nil { - logger.Errorf("error parsing slot from row key %v: %v", r.Key(), err) + logger.Errorf("error parsing validator from row key %v: %v", r.Key(), err) return false } - attesterSlot = max_block_number - attesterSlot + for _, ri := range r[ATTESTATIONS_FAMILY] { + attesterSlot, err := strconv.ParseUint(ri.Column, 10, 64) + if err != nil { + logger.Errorf("error parsing slot from row key %v: %v", r.Key(), err) + return false + } inclusionSlot := max_block_number - uint64(ri.Timestamp)/1000 status := uint64(1) @@ -877,12 +1003,6 @@ func (bigtable *Bigtable) GetValidatorAttestationHistory(validators []uint64, st status = 0 } - validator, err := strconv.ParseUint(strings.TrimPrefix(ri.Column, ATTESTATIONS_FAMILY+":"), 10, 64) - if err != nil { - logger.Errorf("error parsing validator from column key %v: %v", ri.Column, err) - return false - } - if res[validator] == nil { res[validator] = make([]*types.ValidatorAttestation, 0) } @@ -973,8 +1093,6 @@ func (bigtable *Bigtable) GetLastAttestationSlots(validators []uint64) (map[uint } func (bigtable *Bigtable) GetValidatorMissedAttestationHistory(validators []uint64, startEpoch uint64, endEpoch uint64) (map[uint64]map[uint64]bool, error) { - valLen := len(validators) - ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(time.Minute*20)) defer cancel() @@ -988,50 +1106,29 @@ func (bigtable *Bigtable) GetValidatorMissedAttestationHistory(validators []uint return nil, err } - ranges := bigtable.getSlotRanges(startEpoch, endEpoch) + ranges := bigtable.getValidatorsEpochRanges(validators, startEpoch, endEpoch) res := make(map[uint64]map[uint64]bool) foundValid := make(map[uint64]map[uint64]bool) - columnFilters := []gcp_bigtable.Filter{} - if valLen < 1000 { - columnFilters = make([]gcp_bigtable.Filter, 0, len(validators)) - for _, validator := range validators { - columnFilters = append(columnFilters, gcp_bigtable.ColumnFilter(fmt.Sprintf("%d", validator))) - } - } + filter := gcp_bigtable.LatestNFilter(1) - filter := gcp_bigtable.ChainFilters( - gcp_bigtable.FamilyFilter(ATTESTATIONS_FAMILY), - gcp_bigtable.InterleaveFilters(columnFilters...), - gcp_bigtable.LatestNFilter(1), - ) - - if len(columnFilters) == 1 { // special case to retrieve data for one validators - filter = gcp_bigtable.ChainFilters( - gcp_bigtable.FamilyFilter(ATTESTATIONS_FAMILY), - columnFilters[0], - gcp_bigtable.LatestNFilter(1), - ) - } - if len(columnFilters) == 0 { // special case to retrieve data for all validators - filter = gcp_bigtable.ChainFilters( - gcp_bigtable.FamilyFilter(ATTESTATIONS_FAMILY), - gcp_bigtable.LatestNFilter(1), - ) - } - - err = bigtable.tableBeaconchain.ReadRows(ctx, ranges, func(r gcp_bigtable.Row) bool { + err = bigtable.tableValidatorAttestations.ReadRows(ctx, ranges, func(r gcp_bigtable.Row) bool { keySplit := strings.Split(r.Key(), ":") - attesterSlot, err := strconv.ParseUint(keySplit[4], 10, 64) + validator, err := bigtable.validatorKeyToIndex(keySplit[1]) if err != nil { - logger.Errorf("error parsing slot from row key %v: %v", r.Key(), err) + logger.Errorf("error parsing validator from row key %v: %v", r.Key(), err) return false } - attesterSlot = max_block_number - attesterSlot for _, ri := range r[ATTESTATIONS_FAMILY] { + attesterSlot, err := strconv.ParseUint(ri.Column, 10, 64) + if err != nil { + logger.Errorf("error parsing slot from row key %v: %v", r.Key(), err) + return false + } + inclusionSlot := max_block_number - uint64(ri.Timestamp)/1000 status := uint64(1) @@ -1039,12 +1136,6 @@ func (bigtable *Bigtable) GetValidatorMissedAttestationHistory(validators []uint status = 0 } - validator, err := strconv.ParseUint(strings.TrimPrefix(ri.Column, ATTESTATIONS_FAMILY+":"), 10, 64) - if err != nil { - logger.Errorf("error parsing validator from column key %v: %v", ri.Column, err) - return false - } - // only if the attestation was not included in another slot we count it as missed if (status == 0 || orphanedSlotsMap[inclusionSlot]) && (foundValid[validator] == nil || !foundValid[validator][attesterSlot]) { if res[validator] == nil { @@ -1087,40 +1178,23 @@ func (bigtable *Bigtable) GetValidatorSyncDutiesHistory(validators []uint64, sta ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(time.Minute*5)) defer cancel() - ranges := bigtable.getSlotRanges(startEpoch, endEpoch) + ranges := bigtable.getValidatorsEpochRanges(validators, startEpoch, endEpoch) res := make(map[uint64][]*types.ValidatorSyncParticipation, len(validators)) - columnFilters := make([]gcp_bigtable.Filter, 0, len(validators)) - for _, validator := range validators { - columnFilters = append(columnFilters, gcp_bigtable.ColumnFilter(fmt.Sprintf("%d", validator))) - } + filter := gcp_bigtable.LatestNFilter(1) - filter := gcp_bigtable.ChainFilters( - gcp_bigtable.FamilyFilter(SYNC_COMMITTEES_FAMILY), - gcp_bigtable.InterleaveFilters(columnFilters...), - gcp_bigtable.LatestNFilter(1), - ) - - if len(columnFilters) == 1 { // special case to retrieve data for one validators - filter = gcp_bigtable.ChainFilters( - gcp_bigtable.FamilyFilter(SYNC_COMMITTEES_FAMILY), - columnFilters[0], - gcp_bigtable.LatestNFilter(1), - ) - } - if len(columnFilters) == 0 { // special case to retrieve data for all validators - filter = gcp_bigtable.ChainFilters( - gcp_bigtable.FamilyFilter(SYNC_COMMITTEES_FAMILY), - gcp_bigtable.LatestNFilter(1), - ) - } - - err := bigtable.tableBeaconchain.ReadRows(ctx, ranges, func(r gcp_bigtable.Row) bool { + err := bigtable.tableValidatorSyncCommittees.ReadRows(ctx, ranges, func(r gcp_bigtable.Row) bool { for _, ri := range r[SYNC_COMMITTEES_FAMILY] { keySplit := strings.Split(r.Key(), ":") - slot, err := strconv.ParseUint(keySplit[4], 10, 64) + validator, err := bigtable.validatorKeyToIndex(keySplit[1]) + if err != nil { + logger.Errorf("error parsing validator from row key %v: %v", r.Key(), err) + return false + } + + slot, err := strconv.ParseUint(keySplit[3], 10, 64) if err != nil { logger.Errorf("error parsing slot from row key %v: %v", r.Key(), err) return false @@ -1134,12 +1208,6 @@ func (bigtable *Bigtable) GetValidatorSyncDutiesHistory(validators []uint64, sta status = 0 // 0: missed } - validator, err := strconv.ParseUint(strings.TrimPrefix(ri.Column, SYNC_COMMITTEES_FAMILY+":"), 10, 64) - if err != nil { - logger.Errorf("error parsing validator from column key %v: %v", ri.Column, err) - return false - } - if res[validator] == nil { res[validator] = make([]*types.ValidatorSyncParticipation, 0) } @@ -1283,9 +1351,7 @@ func (bigtable *Bigtable) GetValidatorEffectiveness(validators []uint64, epoch u return res, nil } -func (bigtable *Bigtable) GetValidatorBalanceStatistics(startEpoch, endEpoch uint64) (map[uint64]*types.ValidatorBalanceStatistic, error) { - ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(time.Minute*10)) - defer cancel() +func (bigtable *Bigtable) GetValidatorBalanceStatistics(validators []uint64, startEpoch, endEpoch uint64) (map[uint64]*types.ValidatorBalanceStatistic, error) { type ResultContainer struct { mu sync.Mutex @@ -1293,95 +1359,70 @@ func (bigtable *Bigtable) GetValidatorBalanceStatistics(startEpoch, endEpoch uin } resultContainer := ResultContainer{} resultContainer.res = make(map[uint64]*types.ValidatorBalanceStatistic) - g, gCtx := errgroup.WithContext(ctx) - batchSize := utilMath.MaxU64(5, (endEpoch-startEpoch)/5) // we can speed up the loading by splitting it up. Making the batchSize even smaller has no effect but increases the memory consumption. - for e := startEpoch; e < endEpoch; e += batchSize { - fromEpoch := e - toEpoch := fromEpoch + batchSize - 1 - if toEpoch > endEpoch { - toEpoch = endEpoch - } - - g.Go(func() error { - select { - case <-gCtx.Done(): - return gCtx.Err() - default: - } - ranges := bigtable.getEpochRanges(fromEpoch, toEpoch) - err := bigtable.tableBeaconchain.ReadRows(ctx, ranges, func(r gcp_bigtable.Row) bool { - keySplit := strings.Split(r.Key(), ":") - epoch, err := strconv.ParseUint(keySplit[3], 10, 64) - if err != nil { - logger.Errorf("error parsing epoch from row key %v: %v", r.Key(), err) - return false - } - epoch = max_epoch - epoch - logger.Infof("retrieved %v balances entries for epoch %v", len(r[VALIDATOR_BALANCES_FAMILY]), epoch) - resultContainer.mu.Lock() - for _, ri := range r[VALIDATOR_BALANCES_FAMILY] { - validator, err := strconv.ParseUint(strings.TrimPrefix(ri.Column, VALIDATOR_BALANCES_FAMILY+":"), 10, 64) - if err != nil { - logger.Errorf("error parsing validator from column key %v: %v", ri.Column, err) - return false - } - balances := ri.Value + // g, gCtx := errgroup.WithContext(ctx) + batchSize := 100 + // g.SetLimit(1) + for i := 0; i < len(validators); i += batchSize { - balanceBytes := balances[0:8] - effectiveBalanceBytes := balances[8:16] - balance := binary.LittleEndian.Uint64(balanceBytes) - effectiveBalance := binary.LittleEndian.Uint64(effectiveBalanceBytes) - if resultContainer.res[validator] == nil { - resultContainer.res[validator] = &types.ValidatorBalanceStatistic{ - Index: validator, - MinEffectiveBalance: effectiveBalance, - MaxEffectiveBalance: 0, - MinBalance: balance, - MaxBalance: 0, - StartEffectiveBalance: 0, - EndEffectiveBalance: 0, - StartBalance: 0, - EndBalance: 0, - } - } + upperBound := i + batchSize + if len(validators) < upperBound { + upperBound = len(validators) + } + vals := validators[i:upperBound] - if epoch == startEpoch { - resultContainer.res[validator].StartBalance = balance - resultContainer.res[validator].StartEffectiveBalance = effectiveBalance - } + logrus.Infof("retrieving validator balance stats for validators %v - %v", vals[0], vals[len(vals)-1]) - if epoch == endEpoch { - resultContainer.res[validator].EndBalance = balance - resultContainer.res[validator].EndEffectiveBalance = effectiveBalance + res, err := bigtable.GetValidatorBalanceHistory(vals, startEpoch, endEpoch) + if err != nil { + return nil, err + } + resultContainer.mu.Lock() + for validator, balances := range res { + logrus.Infof("processing results for validators %v", validator) + for _, balance := range balances { + if resultContainer.res[validator] == nil { + resultContainer.res[validator] = &types.ValidatorBalanceStatistic{ + Index: validator, + MinEffectiveBalance: balance.EffectiveBalance, + MaxEffectiveBalance: 0, + MinBalance: balance.Balance, + MaxBalance: 0, + StartEffectiveBalance: 0, + EndEffectiveBalance: 0, + StartBalance: 0, + EndBalance: 0, } + } - if balance > resultContainer.res[validator].MaxBalance { - resultContainer.res[validator].MaxBalance = balance - } - if balance < resultContainer.res[validator].MinBalance { - resultContainer.res[validator].MinBalance = balance - } + if balance.Epoch == startEpoch { + resultContainer.res[validator].StartBalance = balance.Balance + resultContainer.res[validator].StartEffectiveBalance = balance.EffectiveBalance + } - if balance > resultContainer.res[validator].MaxEffectiveBalance { - resultContainer.res[validator].MaxEffectiveBalance = balance - } - if balance < resultContainer.res[validator].MinEffectiveBalance { - resultContainer.res[validator].MinEffectiveBalance = balance - } + if balance.Epoch == endEpoch { + resultContainer.res[validator].EndBalance = balance.Balance + resultContainer.res[validator].EndEffectiveBalance = balance.EffectiveBalance } - resultContainer.mu.Unlock() - return true - }, gcp_bigtable.RowFilter(gcp_bigtable.FamilyFilter(VALIDATOR_BALANCES_FAMILY))) + if balance.Balance > resultContainer.res[validator].MaxBalance { + resultContainer.res[validator].MaxBalance = balance.Balance + } + if balance.Balance < resultContainer.res[validator].MinBalance { + resultContainer.res[validator].MinBalance = balance.Balance + } - return err + if balance.EffectiveBalance > resultContainer.res[validator].MaxEffectiveBalance { + resultContainer.res[validator].MaxEffectiveBalance = balance.EffectiveBalance + } + if balance.EffectiveBalance < resultContainer.res[validator].MinEffectiveBalance { + resultContainer.res[validator].MinEffectiveBalance = balance.EffectiveBalance + } + } + } - }) - } + resultContainer.mu.Unlock() - if err := g.Wait(); err != nil { - return nil, err } return resultContainer.res, nil @@ -1391,39 +1432,15 @@ func (bigtable *Bigtable) GetValidatorProposalHistory(validators []uint64, start ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(time.Second*30)) defer cancel() - ranges := bigtable.getSlotRanges(startEpoch, endEpoch) + ranges := bigtable.getValidatorsEpochRanges(validators, startEpoch, endEpoch) res := make(map[uint64][]*types.ValidatorProposal, len(validators)) - columnFilters := make([]gcp_bigtable.Filter, 0, len(validators)) - for _, validator := range validators { - columnFilters = append(columnFilters, gcp_bigtable.ColumnFilter(fmt.Sprintf("%d", validator))) - } - - filter := gcp_bigtable.ChainFilters( - gcp_bigtable.FamilyFilter(PROPOSALS_FAMILY), - gcp_bigtable.InterleaveFilters(columnFilters...), - gcp_bigtable.LatestNFilter(1), - ) - - if len(columnFilters) == 1 { // special case to retrieve data for one validators - filter = gcp_bigtable.ChainFilters( - gcp_bigtable.FamilyFilter(PROPOSALS_FAMILY), - columnFilters[0], - gcp_bigtable.LatestNFilter(1), - ) - } - if len(columnFilters) == 0 { // special case to retrieve data for all validators - filter = gcp_bigtable.ChainFilters( - gcp_bigtable.FamilyFilter(PROPOSALS_FAMILY), - gcp_bigtable.LatestNFilter(1), - ) - } - - err := bigtable.tableBeaconchain.ReadRows(ctx, ranges, func(r gcp_bigtable.Row) bool { + filter := gcp_bigtable.LatestNFilter(1) + err := bigtable.tableValidatorProposals.ReadRows(ctx, ranges, func(r gcp_bigtable.Row) bool { for _, ri := range r[PROPOSALS_FAMILY] { keySplit := strings.Split(r.Key(), ":") - proposalSlot, err := strconv.ParseUint(keySplit[4], 10, 64) + proposalSlot, err := strconv.ParseUint(keySplit[3], 10, 64) if err != nil { logger.Errorf("error parsing slot from row key %v: %v", r.Key(), err) return false @@ -1437,7 +1454,7 @@ func (bigtable *Bigtable) GetValidatorProposalHistory(validators []uint64, start status = 2 } - validator, err := strconv.ParseUint(strings.TrimPrefix(ri.Column, PROPOSALS_FAMILY+":"), 10, 64) + validator, err := bigtable.validatorKeyToIndex(keySplit[1]) if err != nil { logger.Errorf("error parsing validator from column key %v: %v", ri.Column, err) return false @@ -1477,11 +1494,11 @@ func (bigtable *Bigtable) SaveValidatorIncomeDetails(epoch uint64, rewards map[u total := &itypes.ValidatorEpochIncome{} - mut := gcp_bigtable.NewMutation() + muts := make([]*gcp_bigtable.Mutation, 0, 100000) + keys := make([]string, 0, 100000) - muts := 0 + mutsCount := 0 for i, rewardDetails := range rewards { - muts++ data, err := proto.Marshal(rewardDetails) @@ -1489,15 +1506,25 @@ func (bigtable *Bigtable) SaveValidatorIncomeDetails(epoch uint64, rewards map[u return err } - mut.Set(INCOME_DETAILS_COLUMN_FAMILY, fmt.Sprintf("%d", i), ts, data) + mut := &gcp_bigtable.Mutation{} + mut.Set(INCOME_DETAILS_COLUMN_FAMILY, "i", ts, data) + key := fmt.Sprintf("%s:%s:%s", bigtable.chainId, bigtable.validatorIndexToKey(i), reversedPaddedEpoch(epoch)) - if muts%100000 == 0 { - err := bigtable.tableBeaconchain.Apply(ctx, fmt.Sprintf("%s:e:b:%s", bigtable.chainId, reversedPaddedEpoch(epoch)), mut) + muts = append(muts, mut) + keys = append(keys, key) + mutsCount++ + + if mutsCount%100000 == 0 { + errs, err := bigtable.tableValidatorIncomeDetails.ApplyBulk(ctx, keys, muts) if err != nil { return err } - mut = gcp_bigtable.NewMutation() + for _, err := range errs { + return err + } + muts = make([]*gcp_bigtable.Mutation, 0, 100000) + keys = make([]string, 0, 100000) } total.AttestationHeadReward += rewardDetails.AttestationHeadReward @@ -1521,9 +1548,10 @@ func (bigtable *Bigtable) SaveValidatorIncomeDetails(epoch uint64, rewards map[u return err } + mut := &gcp_bigtable.Mutation{} mut.Set(STATS_COLUMN_FAMILY, SUM_COLUMN, ts, sum) - err = bigtable.tableBeaconchain.Apply(ctx, fmt.Sprintf("%s:e:b:%s", bigtable.chainId, reversedPaddedEpoch(epoch)), mut) + err = bigtable.tableValidatorIncomeDetails.Apply(ctx, fmt.Sprintf("%s:%s:%s", bigtable.chainId, SUM_COLUMN, reversedPaddedEpoch(epoch)), mut) if err != nil { return err } @@ -1532,91 +1560,6 @@ func (bigtable *Bigtable) SaveValidatorIncomeDetails(epoch uint64, rewards map[u return nil } -func (bigtable *Bigtable) GetEpochIncomeHistoryDescending(startEpoch uint64, endEpoch uint64) (*itypes.ValidatorEpochIncome, error) { - ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(time.Second*30)) - defer cancel() - - ranges := bigtable.getEpochRanges(startEpoch, endEpoch) - family := gcp_bigtable.FamilyFilter(STATS_COLUMN_FAMILY) - columnFilter := gcp_bigtable.ColumnFilter(SUM_COLUMN) - filter := gcp_bigtable.RowFilter(gcp_bigtable.ChainFilters(family, columnFilter)) - - res := itypes.ValidatorEpochIncome{} - - err := bigtable.tableBeaconchain.ReadRows(ctx, ranges, func(r gcp_bigtable.Row) bool { - if len(r[STATS_COLUMN_FAMILY]) == 0 { - return false - } - err := proto.Unmarshal(r[STATS_COLUMN_FAMILY][0].Value, &res) - if err != nil { - logger.Errorf("error decoding income data for row %v: %v", r.Key(), err) - return false - } - return true - }, filter) - - if err != nil { - return nil, fmt.Errorf("error reading income statistics from bigtable for epoch: %v err: %w", startEpoch, err) - } - - return &res, nil -} - -func (bigtable *Bigtable) GetEpochIncomeHistory(epoch uint64) (*itypes.ValidatorEpochIncome, error) { - - ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(time.Second*30)) - defer cancel() - - key := fmt.Sprintf("%s:e:b:%s", bigtable.chainId, reversedPaddedEpoch(epoch)) - - family := gcp_bigtable.FamilyFilter(STATS_COLUMN_FAMILY) - columnFilter := gcp_bigtable.ColumnFilter(SUM_COLUMN) - filter := gcp_bigtable.RowFilter(gcp_bigtable.ChainFilters(family, columnFilter)) - - row, err := bigtable.tableBeaconchain.ReadRow(ctx, key, filter) - if err != nil { - return nil, fmt.Errorf("error reading income statistics from bigtable for epoch: %v err: %w", epoch, err) - } - - if row != nil { - res := itypes.ValidatorEpochIncome{} - err := proto.Unmarshal(row[STATS_COLUMN_FAMILY][0].Value, &res) - if err != nil { - return nil, fmt.Errorf("error decoding income data for row %v: %w", row.Key(), err) - } - return &res, nil - } - - // if there is no result we have to calculate the sum - income, err := bigtable.GetValidatorIncomeDetailsHistory([]uint64{}, epoch, 1) - if err != nil { - logger.WithError(err).Error("error getting validator income history") - } - - total := &itypes.ValidatorEpochIncome{} - - for _, epochs := range income { - for _, details := range epochs { - total.AttestationHeadReward += details.AttestationHeadReward - total.AttestationSourceReward += details.AttestationSourceReward - total.AttestationSourcePenalty += details.AttestationSourcePenalty - total.AttestationTargetReward += details.AttestationTargetReward - total.AttestationTargetPenalty += details.AttestationTargetPenalty - total.FinalityDelayPenalty += details.FinalityDelayPenalty - total.ProposerSlashingInclusionReward += details.ProposerSlashingInclusionReward - total.ProposerAttestationInclusionReward += details.ProposerAttestationInclusionReward - total.ProposerSyncInclusionReward += details.ProposerSyncInclusionReward - total.SyncCommitteeReward += details.SyncCommitteeReward - total.SyncCommitteePenalty += details.SyncCommitteePenalty - total.SlashingReward += details.SlashingReward - total.SlashingPenalty += details.SlashingPenalty - total.TxFeeRewardWei = utils.AddBigInts(total.TxFeeRewardWei, details.TxFeeRewardWei) - } - } - - return total, nil -} - // GetValidatorIncomeDetailsHistory returns the validator income details // startEpoch & endEpoch are inclusive func (bigtable *Bigtable) GetValidatorIncomeDetailsHistory(validators []uint64, startEpoch uint64, endEpoch uint64) (map[uint64]map[uint64]*itypes.ValidatorEpochIncome, error) { @@ -1627,57 +1570,27 @@ func (bigtable *Bigtable) GetValidatorIncomeDetailsHistory(validators []uint64, ctx, cancel := context.WithTimeout(context.Background(), time.Second*180) defer cancel() - ranges := bigtable.getEpochRanges(startEpoch, endEpoch) + ranges := bigtable.getValidatorsEpochRanges(validators, startEpoch, endEpoch) res := make(map[uint64]map[uint64]*itypes.ValidatorEpochIncome, len(validators)) - valLen := len(validators) + filter := gcp_bigtable.LatestNFilter(1) - // read entire row if you require more than 1000 validators - var columnFilters []gcp_bigtable.Filter - if valLen < 1000 { - columnFilters = make([]gcp_bigtable.Filter, 0, valLen) - for _, validator := range validators { - columnFilters = append(columnFilters, gcp_bigtable.ColumnFilter(fmt.Sprintf("%d", validator))) - } - } - - filter := gcp_bigtable.ChainFilters( - gcp_bigtable.FamilyFilter(INCOME_DETAILS_COLUMN_FAMILY), - gcp_bigtable.InterleaveFilters(columnFilters...), - gcp_bigtable.LatestNFilter(1), - ) - - if len(columnFilters) == 1 { // special case to retrieve data for one validator - filter = gcp_bigtable.ChainFilters( - gcp_bigtable.FamilyFilter(INCOME_DETAILS_COLUMN_FAMILY), - columnFilters[0], - gcp_bigtable.LatestNFilter(1), - ) - } - if len(columnFilters) == 0 { // special case to retrieve data for all validators - filter = gcp_bigtable.ChainFilters( - gcp_bigtable.FamilyFilter(INCOME_DETAILS_COLUMN_FAMILY), - gcp_bigtable.LatestNFilter(1), - ) - } - - err := bigtable.tableBeaconchain.ReadRows(ctx, ranges, func(r gcp_bigtable.Row) bool { + err := bigtable.tableValidatorIncomeDetails.ReadRows(ctx, ranges, func(r gcp_bigtable.Row) bool { keySplit := strings.Split(r.Key(), ":") - epoch, err := strconv.ParseUint(keySplit[3], 10, 64) + validator, err := bigtable.validatorKeyToIndex(keySplit[1]) + if err != nil { + logger.Errorf("error parsing validator from row key %v: %v", r.Key(), err) + return false + } + + epoch, err := strconv.ParseUint(keySplit[2], 10, 64) if err != nil { logger.Errorf("error parsing epoch from row key %v: %v", r.Key(), err) return false } - // logger.Info(max_epoch - epoch) for _, ri := range r[INCOME_DETAILS_COLUMN_FAMILY] { - validator, err := strconv.ParseUint(strings.TrimPrefix(ri.Column, INCOME_DETAILS_COLUMN_FAMILY+":"), 10, 64) - if err != nil { - logger.Errorf("error parsing validator from column key %v: %v", ri.Column, err) - return false - } - incomeDetails := &itypes.ValidatorEpochIncome{} err = proto.Unmarshal(ri.Value, incomeDetails) if err != nil { @@ -1717,45 +1630,18 @@ func (bigtable *Bigtable) GetAggregatedValidatorIncomeDetailsHistory(validators resultContainer := ResultContainer{} resultContainer.res = make(map[uint64]*itypes.ValidatorEpochIncome, len(validators)) - valLen := len(validators) + g, gCtx := errgroup.WithContext(ctx) + batchSize := 100 + g.SetLimit(10) + for i := 0; i < len(validators); i += batchSize { - // read entire row if you require more than 1000 validators - var columnFilters []gcp_bigtable.Filter - if valLen < 1000 { - columnFilters = make([]gcp_bigtable.Filter, 0, valLen) - for _, validator := range validators { - columnFilters = append(columnFilters, gcp_bigtable.ColumnFilter(fmt.Sprintf("%d", validator))) + upperBound := i + batchSize + if len(validators) < upperBound { + upperBound = len(validators) } - } - - filter := gcp_bigtable.ChainFilters( - gcp_bigtable.FamilyFilter(INCOME_DETAILS_COLUMN_FAMILY), - gcp_bigtable.InterleaveFilters(columnFilters...), - gcp_bigtable.LatestNFilter(1), - ) - - if len(columnFilters) == 1 { // special case to retrieve data for one validators - filter = gcp_bigtable.ChainFilters( - gcp_bigtable.FamilyFilter(INCOME_DETAILS_COLUMN_FAMILY), - columnFilters[0], - gcp_bigtable.LatestNFilter(1), - ) - } - if len(columnFilters) == 0 { // special case to retrieve data for all validators - filter = gcp_bigtable.ChainFilters( - gcp_bigtable.FamilyFilter(INCOME_DETAILS_COLUMN_FAMILY), - gcp_bigtable.LatestNFilter(1), - ) - } + vals := validators[i:upperBound] - g, gCtx := errgroup.WithContext(ctx) - batchSize := utilMath.MaxU64(5, (endEpoch-startEpoch)/5) // we can speed up the loading by splitting it up. Making the batchSize even smaller has no effect but increases the memory consumption. - for e := startEpoch; e < endEpoch; e += batchSize { - fromEpoch := e - toEpoch := fromEpoch + batchSize - 1 - if toEpoch > endEpoch { - toEpoch = endEpoch - } + logrus.Infof("retrieving validator income stats for validators %v - %v", vals[0], vals[len(vals)-1]) g.Go(func() error { select { @@ -1763,25 +1649,14 @@ func (bigtable *Bigtable) GetAggregatedValidatorIncomeDetailsHistory(validators return gCtx.Err() default: } - ranges := bigtable.getEpochRanges(fromEpoch, toEpoch) - - err := bigtable.tableBeaconchain.ReadRows(ctx, ranges, func(r gcp_bigtable.Row) bool { - - resultContainer.mu.Lock() - - for _, ri := range r[INCOME_DETAILS_COLUMN_FAMILY] { - validator, err := strconv.ParseUint(strings.TrimPrefix(ri.Column, INCOME_DETAILS_COLUMN_FAMILY+":"), 10, 64) - if err != nil { - logger.Errorf("error parsing validator from column key %v: %v", ri.Column, err) - return false - } + res, err := bigtable.GetValidatorIncomeDetailsHistory(vals, startEpoch, endEpoch) - rewardDetails := &itypes.ValidatorEpochIncome{} - err = proto.Unmarshal(ri.Value, rewardDetails) - if err != nil { - logger.Errorf("error decoding validator income data for row %v: %v", r.Key(), err) - return false - } + if err != nil { + return err + } + resultContainer.mu.Lock() + for validator, epochs := range res { + for _, rewardDetails := range epochs { if resultContainer.res[validator] == nil { resultContainer.res[validator] = &itypes.ValidatorEpochIncome{} @@ -1802,10 +1677,9 @@ func (bigtable *Bigtable) GetAggregatedValidatorIncomeDetailsHistory(validators resultContainer.res[validator].SlashingPenalty += rewardDetails.SlashingPenalty resultContainer.res[validator].TxFeeRewardWei = utils.AddBigInts(resultContainer.res[validator].TxFeeRewardWei, rewardDetails.TxFeeRewardWei) } - resultContainer.mu.Unlock() - return true - }, gcp_bigtable.RowFilter(filter)) - return err + } + resultContainer.mu.Unlock() + return nil }) } @@ -1912,6 +1786,81 @@ func (bigtable *Bigtable) getEpochRanges(startEpoch uint64, endEpoch uint64) gcp return ranges } +func (bigtable *Bigtable) getValidatorEpochRanges(validatorIndex uint64, startEpoch uint64, endEpoch uint64) gcp_bigtable.RowRangeList { + + if endEpoch < startEpoch { // handle overflows + startEpoch = 0 + } + + ranges := gcp_bigtable.RowRangeList{} + if startEpoch == 0 { // special case when the 0 epoch is included + rangeEnd := fmt.Sprintf("%s:%s:%s%s", bigtable.chainId, bigtable.validatorIndexToKey(validatorIndex), reversedPaddedEpoch(0), "\x00") + rangeStart := fmt.Sprintf("%s:%s:%s", bigtable.chainId, bigtable.validatorIndexToKey(validatorIndex), reversedPaddedEpoch(0)) + ranges = append(ranges, gcp_bigtable.NewRange(rangeStart, rangeEnd)) + + // epochs are sorted descending, so start with the largest epoch and end with the smallest + // add \x00 to make the range inclusive + if startEpoch < endEpoch { + rangeEnd = fmt.Sprintf("%s:%s:%s%s", bigtable.chainId, bigtable.validatorIndexToKey(validatorIndex), reversedPaddedEpoch(startEpoch+1), "\x00") + rangeStart = fmt.Sprintf("%s:%s:%s", bigtable.chainId, bigtable.validatorIndexToKey(validatorIndex), reversedPaddedEpoch(endEpoch)) + ranges = append(ranges, gcp_bigtable.NewRange(rangeStart, rangeEnd)) + } + } else { + // epochs are sorted descending, so start with the largest epoch and end with the smallest + // add \x00 to make the range inclusive + rangeEnd := fmt.Sprintf("%s:%s:%s%s", bigtable.chainId, bigtable.validatorIndexToKey(validatorIndex), reversedPaddedEpoch(startEpoch), "\x00") + rangeStart := fmt.Sprintf("%s:%s:%s", bigtable.chainId, bigtable.validatorIndexToKey(validatorIndex), reversedPaddedEpoch(endEpoch)) + ranges = append(ranges, gcp_bigtable.NewRange(rangeStart, rangeEnd)) + } + return ranges +} + +func (bigtable *Bigtable) getValidatorsEpochRanges(validatorIndices []uint64, startEpoch uint64, endEpoch uint64) gcp_bigtable.RowRangeList { + + if endEpoch < startEpoch { // handle overflows + startEpoch = 0 + } + + ranges := make(gcp_bigtable.RowRangeList, 0, int((endEpoch-startEpoch))*len(validatorIndices)) + + for _, validatorIndex := range validatorIndices { + if startEpoch == 0 { // special case when the 0 epoch is included + rangeEnd := fmt.Sprintf("%s:%s:%s%s", bigtable.chainId, bigtable.validatorIndexToKey(validatorIndex), reversedPaddedEpoch(0), "\x00") + rangeStart := fmt.Sprintf("%s:%s:%s", bigtable.chainId, bigtable.validatorIndexToKey(validatorIndex), reversedPaddedEpoch(0)) + ranges = append(ranges, gcp_bigtable.NewRange(rangeStart, rangeEnd)) + + // epochs are sorted descending, so start with the largest epoch and end with the smallest + // add \x00 to make the range inclusive + if startEpoch < endEpoch { + rangeEnd = fmt.Sprintf("%s:%s:%s%s", bigtable.chainId, bigtable.validatorIndexToKey(validatorIndex), reversedPaddedEpoch(startEpoch+1), "\x00") + rangeStart = fmt.Sprintf("%s:%s:%s", bigtable.chainId, bigtable.validatorIndexToKey(validatorIndex), reversedPaddedEpoch(endEpoch)) + ranges = append(ranges, gcp_bigtable.NewRange(rangeStart, rangeEnd)) + } + } else { + // epochs are sorted descending, so start with the largest epoch and end with the smallest + // add \x00 to make the range inclusive + rangeEnd := fmt.Sprintf("%s:%s:%s%s", bigtable.chainId, bigtable.validatorIndexToKey(validatorIndex), reversedPaddedEpoch(startEpoch), "\x00") + rangeStart := fmt.Sprintf("%s:%s:%s", bigtable.chainId, bigtable.validatorIndexToKey(validatorIndex), reversedPaddedEpoch(endEpoch)) + ranges = append(ranges, gcp_bigtable.NewRange(rangeStart, rangeEnd)) + } + } + return ranges +} + +func (bigtable *Bigtable) validatorIndexToKey(index uint64) string { + return utils.ReverseString(fmt.Sprintf("%d", index)) +} + +func (bigtable *Bigtable) validatorKeyToIndex(key string) (uint64, error) { + key = utils.ReverseString(key) + indexKey, err := strconv.ParseUint(key, 10, 64) + + if err != nil { + return 0, err + } + return indexKey, nil +} + func (bigtable *Bigtable) ClearByPrefix(family, prefix string, dryRun bool) ([]string, error) { if family == "" || prefix == "" { return []string{}, fmt.Errorf("please provide family [%v] and prefix [%v]", family, prefix) diff --git a/db/db.go b/db/db.go index 7c0d825acf..9afe9a0c1e 100644 --- a/db/db.go +++ b/db/db.go @@ -1771,6 +1771,13 @@ func UpdateEpochFinalization(finality_epoch uint64) error { return err } +// GetTotalValidatorsCount will return the total-validator-count +func GetValidatorIndices() ([]uint64, error) { + indices := []uint64{} + err := ReaderDb.Select(&indices, "select validatorindex from validators order by validatorindex;") + return indices, err +} + // GetTotalValidatorsCount will return the total-validator-count func GetTotalValidatorsCount() (uint64, error) { var totalCount uint64 diff --git a/db/statistics.go b/db/statistics.go index ba13077f02..9064da0287 100644 --- a/db/statistics.go +++ b/db/statistics.go @@ -73,6 +73,11 @@ func WriteValidatorStatisticsForDay(day uint64, concurrencyTotal uint64, concurr } logger.Infof("getting exported state took %v", time.Since(start)) + validators, err := GetValidatorIndices() + if err != nil { + return err + } + if exported.FailedAttestations && exported.SyncDuties && exported.WithdrawalsDeposits && exported.Balance && exported.ClRewards && exported.ElRewards && exported.TotalAccumulation && exported.TotalPerformance && exported.BlockStats && exported.Status { logger.Infof("Skipping day %v as it is already exported", day) return nil @@ -80,13 +85,13 @@ func WriteValidatorStatisticsForDay(day uint64, concurrencyTotal uint64, concurr if exported.FailedAttestations { logger.Infof("Skipping failed attestations") - } else if err := WriteValidatorFailedAttestationsStatisticsForDay(day, concurrencyFailedAttestations); err != nil { + } else if err := WriteValidatorFailedAttestationsStatisticsForDay(validators, day, concurrencyFailedAttestations); err != nil { return fmt.Errorf("error in WriteValidatorFailedAttestationsStatisticsForDay: %w", err) } if exported.SyncDuties { logger.Infof("Skipping sync duties") - } else if err := WriteValidatorSyncDutiesForDay(day); err != nil { + } else if err := WriteValidatorSyncDutiesForDay(validators, day); err != nil { return fmt.Errorf("error in WriteValidatorSyncDutiesForDay: %w", err) } @@ -104,13 +109,13 @@ func WriteValidatorStatisticsForDay(day uint64, concurrencyTotal uint64, concurr if exported.Balance { logger.Infof("Skipping balances") - } else if err := WriteValidatorBalances(day); err != nil { + } else if err := WriteValidatorBalances(validators, day); err != nil { return fmt.Errorf("error in WriteValidatorBalances: %w", err) } if exported.ClRewards { logger.Infof("Skipping cl rewards") - } else if err := WriteValidatorClIcome(day, concurrencyCl); err != nil { + } else if err := WriteValidatorClIcome(validators, day, concurrencyCl); err != nil { return fmt.Errorf("error in WriteValidatorClIcome: %w", err) } @@ -665,7 +670,7 @@ func WriteValidatorElIcome(day uint64) error { return nil } -func WriteValidatorClIcome(day uint64, concurrency uint64) error { +func WriteValidatorClIcome(validators []uint64, day uint64, concurrency uint64) error { ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(time.Minute*10)) defer cancel() exportStart := time.Now() @@ -704,7 +709,7 @@ func WriteValidatorClIcome(day uint64, concurrency uint64) error { firstEpoch, lastEpoch := utils.GetFirstAndLastEpochForDay(day) logger.Infof("exporting cl_rewards_wei statistics") - incomeStats, err := BigtableClient.GetAggregatedValidatorIncomeDetailsHistory([]uint64{}, firstEpoch, lastEpoch) + incomeStats, err := BigtableClient.GetAggregatedValidatorIncomeDetailsHistory(validators, firstEpoch, lastEpoch) if err != nil { return fmt.Errorf("error in GetAggregatedValidatorIncomeDetailsHistory for firstEpoch [%v] and lastEpoch [%v]: %w", firstEpoch, lastEpoch, err) } @@ -809,7 +814,7 @@ func WriteValidatorClIcome(day uint64, concurrency uint64) error { return nil } -func WriteValidatorBalances(day uint64) error { +func WriteValidatorBalances(validators []uint64, day uint64) error { ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(time.Minute*10)) defer cancel() @@ -827,7 +832,7 @@ func WriteValidatorBalances(day uint64) error { start := time.Now() logger.Infof("exporting min_balance, max_balance, min_effective_balance, max_effective_balance, start_balance, start_effective_balance, end_balance and end_effective_balance statistics") - balanceStatistics, err := BigtableClient.GetValidatorBalanceStatistics(firstEpoch, lastEpoch) + balanceStatistics, err := BigtableClient.GetValidatorBalanceStatistics(validators, firstEpoch, lastEpoch) if err != nil { return fmt.Errorf("error in GetValidatorBalanceStatistics for firstEpoch [%v] and lastEpoch [%v]: %w", firstEpoch, lastEpoch, err) } @@ -1025,7 +1030,7 @@ func WriteValidatorDepositWithdrawals(day uint64) error { return nil } -func WriteValidatorSyncDutiesForDay(day uint64) error { +func WriteValidatorSyncDutiesForDay(validators []uint64, day uint64) error { exportStart := time.Now() defer func() { metrics.TaskDuration.WithLabelValues("db_update_validator_sync_stats").Observe(time.Since(exportStart).Seconds()) @@ -1040,7 +1045,7 @@ func WriteValidatorSyncDutiesForDay(day uint64) error { start := time.Now() logrus.Infof("Update Sync duties for day [%v] epoch %v -> %v", day, startEpoch, endEpoch) - syncStats, err := BigtableClient.GetValidatorSyncDutiesStatistics([]uint64{}, startEpoch, endEpoch) + syncStats, err := BigtableClient.GetValidatorSyncDutiesStatistics(validators, startEpoch, endEpoch) if err != nil { return fmt.Errorf("error in GetValidatorSyncDutiesStatistics for startEpoch [%v] and endEpoch [%v]: %w", startEpoch, endEpoch, err) } @@ -1104,7 +1109,7 @@ func WriteValidatorSyncDutiesForDay(day uint64) error { return nil } -func WriteValidatorFailedAttestationsStatisticsForDay(day uint64, concurrency uint64) error { +func WriteValidatorFailedAttestationsStatisticsForDay(validators []uint64, day uint64, concurrency uint64) error { ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(time.Minute*10)) defer cancel() exportStart := time.Now() @@ -1123,31 +1128,34 @@ func WriteValidatorFailedAttestationsStatisticsForDay(day uint64, concurrency ui logrus.Infof("exporting 'failed attestations' statistics firstEpoch: %v lastEpoch: %v", firstEpoch, lastEpoch) // first key is the batch start index and the second is the validator id - failed := map[uint64]map[uint64]*types.ValidatorMissedAttestationsStatistic{} + validatorMap := map[uint64]*types.ValidatorMissedAttestationsStatistic{} mux := sync.Mutex{} g, gCtx := errgroup.WithContext(ctx) - g.SetLimit(int(concurrency)) - epochBatchSize := uint64(2) // Fetching 2 Epochs per batch seems to be the fastest way to go - for i := firstEpoch; i < lastEpoch; i += epochBatchSize { - fromEpoch := i - toEpoch := fromEpoch + epochBatchSize - if toEpoch >= lastEpoch { - toEpoch = lastEpoch - } else { - toEpoch-- + batchSize := 100 + g.SetLimit(10) + for i := 0; i < len(validators); i += batchSize { + + upperBound := i + batchSize + if len(validators) < upperBound { + upperBound = len(validators) } + vals := validators[i:upperBound] + + logrus.Infof("retrieving validator missed attestations stats for validators %v - %v", vals[0], vals[len(vals)-1]) g.Go(func() error { select { case <-gCtx.Done(): return gCtx.Err() default: } - ma, err := BigtableClient.GetValidatorMissedAttestationsCount([]uint64{}, fromEpoch, toEpoch) + ma, err := BigtableClient.GetValidatorMissedAttestationsCount(vals, firstEpoch, lastEpoch) if err != nil { - return fmt.Errorf("error in GetValidatorMissedAttestationsCount for fromEpoch [%v] and toEpoch [%v]: %w", fromEpoch, toEpoch, err) + return fmt.Errorf("error in GetValidatorMissedAttestationsCount for fromEpoch [%v] and toEpoch [%v]: %w", firstEpoch, lastEpoch, err) } mux.Lock() - failed[fromEpoch] = ma + for validator, stats := range ma { + validatorMap[validator] = stats + } mux.Unlock() return nil }) @@ -1157,18 +1165,6 @@ func WriteValidatorFailedAttestationsStatisticsForDay(day uint64, concurrency ui return err } - validatorMap := map[uint64]*types.ValidatorMissedAttestationsStatistic{} - for _, f := range failed { - - for key, val := range f { - if validatorMap[key] == nil { - validatorMap[key] = val - } else { - validatorMap[key].MissedAttestations += val.MissedAttestations - } - } - } - logrus.Infof("fetching 'failed attestations' done in %v, now we export them to the db", time.Since(start)) start = time.Now() maArr := make([]*types.ValidatorMissedAttestationsStatistic, 0, len(validatorMap)) @@ -1179,11 +1175,11 @@ func WriteValidatorFailedAttestationsStatisticsForDay(day uint64, concurrency ui g, gCtx = errgroup.WithContext(ctx) - batchSize := 100 // max: 65535 / 4, but we are faster with smaller batches - for b := 0; b < len(maArr); b += batchSize { + dbBatchSize := 100 // max: 65535 / 4, but we are faster with smaller batches + for b := 0; b < len(maArr); b += dbBatchSize { start := b - end := b + batchSize + end := b + dbBatchSize if len(maArr) < end { end = len(maArr) } diff --git a/handlers/validator.go b/handlers/validator.go index 7c1f45a195..c73e04f94f 100644 --- a/handlers/validator.go +++ b/handlers/validator.go @@ -1997,7 +1997,7 @@ func ValidatorSync(w http.ResponseWriter, r *http.Request) { startEpoch = firstShownEpoch - (limit - 1) endEpoch = firstShownEpoch } - syncDuties, err := db.BigtableClient.GetValidatorSyncDutiesHistoryOrdered([]uint64{}, startEpoch, endEpoch, ascOrdering) + syncDuties, err := db.BigtableClient.GetValidatorSyncDutiesHistoryOrdered([]uint64{validatorIndex}, startEpoch, endEpoch, ascOrdering) if err != nil { logger.Errorf("error retrieving validator sync duty data from bigtable: %v", err) http.Error(w, "Internal server error", http.StatusInternalServerError) @@ -2027,7 +2027,7 @@ func ValidatorSync(w http.ResponseWriter, r *http.Request) { startEpoch = lastShownEpoch endEpoch = lastShownEpoch + nextPeriodLimit - 1 } - nextPeriodSyncDuties, err := db.BigtableClient.GetValidatorSyncDutiesHistoryOrdered([]uint64{}, startEpoch, endEpoch, ascOrdering) + nextPeriodSyncDuties, err := db.BigtableClient.GetValidatorSyncDutiesHistoryOrdered([]uint64{validatorIndex}, startEpoch, endEpoch, ascOrdering) if err != nil { logger.Errorf("error retrieving second validator sync duty data from bigtable: %v", err) http.Error(w, "Internal server error", http.StatusInternalServerError) diff --git a/services/charts_updater.go b/services/charts_updater.go index 614d7893e2..a96d502f9c 100644 --- a/services/charts_updater.go +++ b/services/charts_updater.go @@ -599,7 +599,12 @@ func balanceDistributionChartData() (*types.GenericChartData, error) { return nil, fmt.Errorf("chart-data not available pre-genesis") } - balances, err := db.BigtableClient.GetValidatorBalanceHistory([]uint64{}, epoch, epoch) + validators, err := db.GetValidatorIndices() + if err != nil { + return nil, err + } + + balances, err := db.BigtableClient.GetValidatorBalanceHistory(validators, epoch, epoch) if err != nil { return nil, err } @@ -649,7 +654,12 @@ func effectiveBalanceDistributionChartData() (*types.GenericChartData, error) { return nil, fmt.Errorf("chart-data not available pre-genesis") } - balances, err := db.BigtableClient.GetValidatorBalanceHistory([]uint64{}, epoch, epoch) + validators, err := db.GetValidatorIndices() + if err != nil { + return nil, err + } + + balances, err := db.BigtableClient.GetValidatorBalanceHistory(validators, epoch, epoch) if err != nil { return nil, err } diff --git a/services/notifications.go b/services/notifications.go index 1deaedf544..5f38797927 100644 --- a/services/notifications.go +++ b/services/notifications.go @@ -1375,7 +1375,13 @@ func collectAttestationAndOfflineValidatorNotifications(notificationsByUserID ma } // get attestations for all validators for the last 4 epochs - attestations, err := db.BigtableClient.GetValidatorAttestationHistory([]uint64{}, epoch-3, epoch) + + validators, err := db.GetValidatorIndices() + if err != nil { + return err + } + + attestations, err := db.BigtableClient.GetValidatorAttestationHistory(validators, epoch-3, epoch) if err != nil { return fmt.Errorf("error getting validator attestations from bigtable %w", err) } diff --git a/services/services.go b/services/services.go index 4e435c05cd..40c779427f 100644 --- a/services/services.go +++ b/services/services.go @@ -1548,7 +1548,13 @@ func getBurnPageData() (*types.BurnPageData, error) { // } // swap this for GetEpochIncomeHistory in the future - income, err := db.BigtableClient.GetValidatorIncomeDetailsHistory([]uint64{}, latestEpoch-10, latestBlock) + + validators, err := db.GetValidatorIndices() + if err != nil { + return nil, err + } + + income, err := db.BigtableClient.GetValidatorIncomeDetailsHistory(validators, latestEpoch-10, latestEpoch) if err != nil { logger.WithError(err).Error("error getting validator income history") } diff --git a/utils/utils.go b/utils/utils.go index 7f1d072982..028c212d52 100644 --- a/utils/utils.go +++ b/utils/utils.go @@ -1387,3 +1387,11 @@ func SortedUniqueUint64(arr []uint64) []uint64 { return result } + +func ReverseString(s string) string { + runes := []rune(s) + for i, j := 0, len(runes)-1; i < j; i, j = i+1, j-1 { + runes[i], runes[j] = runes[j], runes[i] + } + return string(runes) +} From f0364c5af9628aef46a4852b1b263882ed981ba8 Mon Sep 17 00:00:00 2001 From: peter <1674920+peterbitfly@users.noreply.github.com> Date: Mon, 21 Aug 2023 11:21:28 +0200 Subject: [PATCH 002/106] (BIDS-2379) implement parallel fetching for all cl bigtable read requests --- db/bigtable.go | 776 +++++++++++++++++++++++------------------------ db/statistics.go | 37 +-- 2 files changed, 387 insertions(+), 426 deletions(-) diff --git a/db/bigtable.go b/db/bigtable.go index b1420e8ed6..5f9deb1de2 100644 --- a/db/bigtable.go +++ b/db/bigtable.go @@ -810,60 +810,6 @@ func (bigtable *Bigtable) SaveSyncComitteeDuties(blocks map[uint64]map[string]*t return nil } -func (bigtable *Bigtable) GetSingleValidatorBalanceHistory(validator uint64, startEpoch uint64, endEpoch uint64) ([]*types.ValidatorBalance, error) { - - if endEpoch < startEpoch { // handle overflows - startEpoch = 0 - } - - ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(time.Second*30)) - defer cancel() - - ranges := bigtable.getValidatorEpochRanges(validator, startEpoch, endEpoch) - res := make([]*types.ValidatorBalance, 0, endEpoch-startEpoch) - - ro := gcp_bigtable.LimitRows(int64(endEpoch - startEpoch + 1)) - handleRow := func(r gcp_bigtable.Row) bool { - keySplit := strings.Split(r.Key(), ":") - - epoch, err := strconv.ParseUint(keySplit[2], 10, 64) - if err != nil { - logger.Errorf("error parsing epoch from row key %v: %v", r.Key(), err) - return false - } - validator, err := bigtable.validatorKeyToIndex(keySplit[1]) - if err != nil { - logger.Errorf("error parsing validator index from row key %v: %v", r.Key(), err) - return false - } - - for _, ri := range r[VALIDATOR_BALANCES_FAMILY] { - balances := ri.Value - - balanceBytes := balances[0:8] - effectiveBalanceBytes := balances[8:16] - balance := binary.LittleEndian.Uint64(balanceBytes) - effectiveBalance := binary.LittleEndian.Uint64(effectiveBalanceBytes) - - res = append(res, &types.ValidatorBalance{ - Epoch: max_epoch - epoch, - Balance: balance, - EffectiveBalance: effectiveBalance, - Index: validator, - PublicKey: []byte{}, - }) - } - return true - } - - err := bigtable.tableValidatorBalances.ReadRows(ctx, ranges, handleRow, ro) - if err != nil { - return nil, err - } - - return res, nil -} - func (bigtable *Bigtable) GetValidatorBalanceHistory(validators []uint64, startEpoch uint64, endEpoch uint64) (map[uint64][]*types.ValidatorBalance, error) { if len(validators) == 0 { @@ -873,7 +819,7 @@ func (bigtable *Bigtable) GetValidatorBalanceHistory(validators []uint64, startE batchSize := 1000 concurrency := 10 - ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(time.Minute*5)) + ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(time.Minute)) defer cancel() res := make(map[uint64][]*types.ValidatorBalance, len(validators)) @@ -946,7 +892,7 @@ func (bigtable *Bigtable) GetValidatorBalanceHistory(validators []uint64, startE return err } - logrus.Infof("retrieved data for validators %v - %v", vals[0], vals[len(vals)-1]) + // logrus.Infof("retrieved data for validators %v - %v", vals[0], vals[len(vals)-1]) return nil }) } @@ -960,6 +906,13 @@ func (bigtable *Bigtable) GetValidatorBalanceHistory(validators []uint64, startE func (bigtable *Bigtable) GetValidatorAttestationHistory(validators []uint64, startEpoch uint64, endEpoch uint64) (map[uint64][]*types.ValidatorAttestation, error) { + if len(validators) == 0 { + return nil, fmt.Errorf("passing empty validator array is unsupported") + } + + batchSize := 1000 + concurrency := 10 + ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(time.Minute*5)) defer cancel() @@ -973,63 +926,89 @@ func (bigtable *Bigtable) GetValidatorAttestationHistory(validators []uint64, st return nil, err } - ranges := bigtable.getValidatorsEpochRanges(validators, startEpoch, endEpoch) res := make(map[uint64][]*types.ValidatorAttestation, len(validators)) + resMux := &sync.Mutex{} filter := gcp_bigtable.LatestNFilter(1) - err = bigtable.tableValidatorAttestations.ReadRows(ctx, ranges, func(r gcp_bigtable.Row) bool { - keySplit := strings.Split(r.Key(), ":") + g, gCtx := errgroup.WithContext(ctx) + g.SetLimit(concurrency) - validator, err := bigtable.validatorKeyToIndex(keySplit[1]) - if err != nil { - logger.Errorf("error parsing validator from row key %v: %v", r.Key(), err) - return false + for i := 0; i < len(validators); i += batchSize { + + upperBound := i + batchSize + if len(validators) < upperBound { + upperBound = len(validators) } + vals := validators[i:upperBound] - for _, ri := range r[ATTESTATIONS_FAMILY] { - attesterSlot, err := strconv.ParseUint(ri.Column, 10, 64) - if err != nil { - logger.Errorf("error parsing slot from row key %v: %v", r.Key(), err) - return false - } - inclusionSlot := max_block_number - uint64(ri.Timestamp)/1000 - - status := uint64(1) - if inclusionSlot == max_block_number { - inclusionSlot = 0 - status = 0 - } else if orphanedSlotsMap[inclusionSlot] { - status = 0 + g.Go(func() error { + select { + case <-gCtx.Done(): + return gCtx.Err() + default: } + ranges := bigtable.getValidatorsEpochRanges(vals, startEpoch, endEpoch) + err = bigtable.tableValidatorAttestations.ReadRows(ctx, ranges, func(r gcp_bigtable.Row) bool { + keySplit := strings.Split(r.Key(), ":") - if res[validator] == nil { - res[validator] = make([]*types.ValidatorAttestation, 0) - } + validator, err := bigtable.validatorKeyToIndex(keySplit[1]) + if err != nil { + logger.Errorf("error parsing validator from row key %v: %v", r.Key(), err) + return false + } + + for _, ri := range r[ATTESTATIONS_FAMILY] { + attesterSlotString := strings.Replace(ri.Column, ATTESTATIONS_FAMILY+":", "", 1) + attesterSlot, err := strconv.ParseUint(attesterSlotString, 10, 64) + if err != nil { + logger.Errorf("error parsing slot from row key %v: %v", r.Key(), err) + return false + } + inclusionSlot := max_block_number - uint64(ri.Timestamp)/1000 + + status := uint64(1) + if inclusionSlot == max_block_number { + inclusionSlot = 0 + status = 0 + } else if orphanedSlotsMap[inclusionSlot] { + status = 0 + } + + resMux.Lock() + if res[validator] == nil { + res[validator] = make([]*types.ValidatorAttestation, 0) + } + + if len(res[validator]) > 0 && res[validator][len(res[validator])-1].AttesterSlot == attesterSlot { + // don't override successful attestion, that was included in a different slot + if status == 1 || res[validator][len(res[validator])-1].Status != 1 { + res[validator][len(res[validator])-1].InclusionSlot = inclusionSlot + res[validator][len(res[validator])-1].Status = status + res[validator][len(res[validator])-1].Delay = int64(inclusionSlot - attesterSlot) + } + } else { + res[validator] = append(res[validator], &types.ValidatorAttestation{ + Index: validator, + Epoch: attesterSlot / utils.Config.Chain.Config.SlotsPerEpoch, + AttesterSlot: attesterSlot, + CommitteeIndex: 0, + Status: status, + InclusionSlot: inclusionSlot, + Delay: int64(inclusionSlot) - int64(attesterSlot) - 1, + }) + } + resMux.Unlock() - if len(res[validator]) > 0 && res[validator][len(res[validator])-1].AttesterSlot == attesterSlot { - // don't override successful attestion, that was included in a different slot - if status == 1 || res[validator][len(res[validator])-1].Status != 1 { - res[validator][len(res[validator])-1].InclusionSlot = inclusionSlot - res[validator][len(res[validator])-1].Status = status - res[validator][len(res[validator])-1].Delay = int64(inclusionSlot - attesterSlot) } - } else { - res[validator] = append(res[validator], &types.ValidatorAttestation{ - Index: validator, - Epoch: attesterSlot / utils.Config.Chain.Config.SlotsPerEpoch, - AttesterSlot: attesterSlot, - CommitteeIndex: 0, - Status: status, - InclusionSlot: inclusionSlot, - Delay: int64(inclusionSlot) - int64(attesterSlot) - 1, - }) - } + return true + }, gcp_bigtable.RowFilter(filter)) - } - return true - }, gcp_bigtable.RowFilter(filter)) - if err != nil { + return err + }) + } + + if err := g.Wait(); err != nil { return nil, err } @@ -1093,6 +1072,14 @@ func (bigtable *Bigtable) GetLastAttestationSlots(validators []uint64) (map[uint } func (bigtable *Bigtable) GetValidatorMissedAttestationHistory(validators []uint64, startEpoch uint64, endEpoch uint64) (map[uint64]map[uint64]bool, error) { + + if len(validators) == 0 { + return nil, fmt.Errorf("passing empty validator array is unsupported") + } + + batchSize := 1000 + concurrency := 10 + ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(time.Minute*20)) defer cancel() @@ -1106,55 +1093,80 @@ func (bigtable *Bigtable) GetValidatorMissedAttestationHistory(validators []uint return nil, err } - ranges := bigtable.getValidatorsEpochRanges(validators, startEpoch, endEpoch) - res := make(map[uint64]map[uint64]bool) foundValid := make(map[uint64]map[uint64]bool) + resMux := &sync.Mutex{} + filter := gcp_bigtable.LatestNFilter(1) - err = bigtable.tableValidatorAttestations.ReadRows(ctx, ranges, func(r gcp_bigtable.Row) bool { - keySplit := strings.Split(r.Key(), ":") + g, gCtx := errgroup.WithContext(ctx) + g.SetLimit(concurrency) - validator, err := bigtable.validatorKeyToIndex(keySplit[1]) - if err != nil { - logger.Errorf("error parsing validator from row key %v: %v", r.Key(), err) - return false + for i := 0; i < len(validators); i += batchSize { + + upperBound := i + batchSize + if len(validators) < upperBound { + upperBound = len(validators) } + vals := validators[i:upperBound] - for _, ri := range r[ATTESTATIONS_FAMILY] { - attesterSlot, err := strconv.ParseUint(ri.Column, 10, 64) - if err != nil { - logger.Errorf("error parsing slot from row key %v: %v", r.Key(), err) - return false + g.Go(func() error { + select { + case <-gCtx.Done(): + return gCtx.Err() + default: } + ranges := bigtable.getValidatorsEpochRanges(vals, startEpoch, endEpoch) + err = bigtable.tableValidatorAttestations.ReadRows(ctx, ranges, func(r gcp_bigtable.Row) bool { + keySplit := strings.Split(r.Key(), ":") - inclusionSlot := max_block_number - uint64(ri.Timestamp)/1000 + validator, err := bigtable.validatorKeyToIndex(keySplit[1]) + if err != nil { + logger.Errorf("error parsing validator from row key %v: %v", r.Key(), err) + return false + } - status := uint64(1) - if inclusionSlot == max_block_number { - status = 0 - } + for _, ri := range r[ATTESTATIONS_FAMILY] { + attesterSlot, err := strconv.ParseUint(ri.Column, 10, 64) + if err != nil { + logger.Errorf("error parsing slot from row key %v: %v", r.Key(), err) + return false + } - // only if the attestation was not included in another slot we count it as missed - if (status == 0 || orphanedSlotsMap[inclusionSlot]) && (foundValid[validator] == nil || !foundValid[validator][attesterSlot]) { - if res[validator] == nil { - res[validator] = make(map[uint64]bool, 0) - } - res[validator][attesterSlot] = true - } else { - if res[validator] != nil && res[validator][attesterSlot] { - delete(res[validator], attesterSlot) - } - if foundValid[validator] == nil { - foundValid[validator] = make(map[uint64]bool, 0) + inclusionSlot := max_block_number - uint64(ri.Timestamp)/1000 + + status := uint64(1) + if inclusionSlot == max_block_number { + status = 0 + } + + resMux.Lock() + // only if the attestation was not included in another slot we count it as missed + if (status == 0 || orphanedSlotsMap[inclusionSlot]) && (foundValid[validator] == nil || !foundValid[validator][attesterSlot]) { + if res[validator] == nil { + res[validator] = make(map[uint64]bool, 0) + } + res[validator][attesterSlot] = true + } else { + if res[validator] != nil && res[validator][attesterSlot] { + delete(res[validator], attesterSlot) + } + if foundValid[validator] == nil { + foundValid[validator] = make(map[uint64]bool, 0) + } + foundValid[validator][attesterSlot] = true + } + resMux.Unlock() } - foundValid[validator][attesterSlot] = true - } - } - return true - }, gcp_bigtable.RowFilter(filter)) - if err != nil { + return true + }, gcp_bigtable.RowFilter(filter)) + + return err + }) + } + + if err := g.Wait(); err != nil { return nil, err } @@ -1175,56 +1187,89 @@ func (bigtable *Bigtable) GetValidatorSyncDutiesHistoryOrdered(validators []uint } func (bigtable *Bigtable) GetValidatorSyncDutiesHistory(validators []uint64, startEpoch uint64, endEpoch uint64) (map[uint64][]*types.ValidatorSyncParticipation, error) { + + if len(validators) == 0 { + return nil, fmt.Errorf("passing empty validator array is unsupported") + } + + batchSize := 1000 + concurrency := 10 + ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(time.Minute*5)) defer cancel() - ranges := bigtable.getValidatorsEpochRanges(validators, startEpoch, endEpoch) res := make(map[uint64][]*types.ValidatorSyncParticipation, len(validators)) + resMux := &sync.Mutex{} filter := gcp_bigtable.LatestNFilter(1) - err := bigtable.tableValidatorSyncCommittees.ReadRows(ctx, ranges, func(r gcp_bigtable.Row) bool { + g, gCtx := errgroup.WithContext(ctx) + g.SetLimit(concurrency) - for _, ri := range r[SYNC_COMMITTEES_FAMILY] { - keySplit := strings.Split(r.Key(), ":") + for i := 0; i < len(validators); i += batchSize { - validator, err := bigtable.validatorKeyToIndex(keySplit[1]) - if err != nil { - logger.Errorf("error parsing validator from row key %v: %v", r.Key(), err) - return false - } + upperBound := i + batchSize + if len(validators) < upperBound { + upperBound = len(validators) + } + vals := validators[i:upperBound] - slot, err := strconv.ParseUint(keySplit[3], 10, 64) - if err != nil { - logger.Errorf("error parsing slot from row key %v: %v", r.Key(), err) - return false + g.Go(func() error { + select { + case <-gCtx.Done(): + return gCtx.Err() + default: } - slot = max_block_number - slot - inclusionSlot := max_block_number - uint64(ri.Timestamp)/1000 + ranges := bigtable.getValidatorsEpochRanges(vals, startEpoch, endEpoch) + err := bigtable.tableValidatorSyncCommittees.ReadRows(ctx, ranges, func(r gcp_bigtable.Row) bool { - status := uint64(1) // 1: participated - if inclusionSlot == max_block_number { - inclusionSlot = 0 - status = 0 // 0: missed - } + for _, ri := range r[SYNC_COMMITTEES_FAMILY] { + keySplit := strings.Split(r.Key(), ":") - if res[validator] == nil { - res[validator] = make([]*types.ValidatorSyncParticipation, 0) - } + validator, err := bigtable.validatorKeyToIndex(keySplit[1]) + if err != nil { + logger.Errorf("error parsing validator from row key %v: %v", r.Key(), err) + return false + } - if len(res[validator]) > 0 && res[validator][len(res[validator])-1].Slot == slot { - res[validator][len(res[validator])-1].Status = status - } else { - res[validator] = append(res[validator], &types.ValidatorSyncParticipation{ - Slot: slot, - Status: status, - }) - } + slot, err := strconv.ParseUint(keySplit[3], 10, 64) + if err != nil { + logger.Errorf("error parsing slot from row key %v: %v", r.Key(), err) + return false + } + slot = max_block_number - slot + inclusionSlot := max_block_number - uint64(ri.Timestamp)/1000 - } - return true - }, gcp_bigtable.RowFilter(filter)) - if err != nil { + status := uint64(1) // 1: participated + if inclusionSlot == max_block_number { + inclusionSlot = 0 + status = 0 // 0: missed + } + + resMux.Lock() + if res[validator] == nil { + res[validator] = make([]*types.ValidatorSyncParticipation, 0) + } + + if len(res[validator]) > 0 && res[validator][len(res[validator])-1].Slot == slot { + res[validator][len(res[validator])-1].Status = status + } else { + res[validator] = append(res[validator], &types.ValidatorSyncParticipation{ + Slot: slot, + Status: status, + }) + } + resMux.Unlock() + + } + return true + }, gcp_bigtable.RowFilter(filter)) + + return err + }) + } + + if err := g.Wait(); err != nil { return nil, err } @@ -1361,7 +1406,7 @@ func (bigtable *Bigtable) GetValidatorBalanceStatistics(validators []uint64, sta resultContainer.res = make(map[uint64]*types.ValidatorBalanceStatistic) // g, gCtx := errgroup.WithContext(ctx) - batchSize := 100 + batchSize := 10000 // g.SetLimit(1) for i := 0; i < len(validators); i += batchSize { @@ -1379,7 +1424,6 @@ func (bigtable *Bigtable) GetValidatorBalanceStatistics(validators []uint64, sta } resultContainer.mu.Lock() for validator, balances := range res { - logrus.Infof("processing results for validators %v", validator) for _, balance := range balances { if resultContainer.res[validator] == nil { resultContainer.res[validator] = &types.ValidatorBalanceStatistic{ @@ -1429,56 +1473,89 @@ func (bigtable *Bigtable) GetValidatorBalanceStatistics(validators []uint64, sta } func (bigtable *Bigtable) GetValidatorProposalHistory(validators []uint64, startEpoch uint64, endEpoch uint64) (map[uint64][]*types.ValidatorProposal, error) { + if len(validators) == 0 { + return nil, fmt.Errorf("passing empty validator array is unsupported") + } + + batchSize := 1000 + concurrency := 10 + ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(time.Second*30)) defer cancel() - ranges := bigtable.getValidatorsEpochRanges(validators, startEpoch, endEpoch) res := make(map[uint64][]*types.ValidatorProposal, len(validators)) + resMux := &sync.Mutex{} filter := gcp_bigtable.LatestNFilter(1) - err := bigtable.tableValidatorProposals.ReadRows(ctx, ranges, func(r gcp_bigtable.Row) bool { - for _, ri := range r[PROPOSALS_FAMILY] { - keySplit := strings.Split(r.Key(), ":") - proposalSlot, err := strconv.ParseUint(keySplit[3], 10, 64) - if err != nil { - logger.Errorf("error parsing slot from row key %v: %v", r.Key(), err) - return false - } - proposalSlot = max_block_number - proposalSlot - inclusionSlot := max_block_number - uint64(r[PROPOSALS_FAMILY][0].Timestamp)/1000 + g, gCtx := errgroup.WithContext(ctx) + g.SetLimit(concurrency) - status := uint64(1) - if inclusionSlot == max_block_number { - inclusionSlot = 0 - status = 2 - } + for i := 0; i < len(validators); i += batchSize { - validator, err := bigtable.validatorKeyToIndex(keySplit[1]) - if err != nil { - logger.Errorf("error parsing validator from column key %v: %v", ri.Column, err) - return false - } + upperBound := i + batchSize + if len(validators) < upperBound { + upperBound = len(validators) + } + vals := validators[i:upperBound] - if res[validator] == nil { - res[validator] = make([]*types.ValidatorProposal, 0) + g.Go(func() error { + select { + case <-gCtx.Done(): + return gCtx.Err() + default: } + ranges := bigtable.getValidatorsEpochRanges(vals, startEpoch, endEpoch) + err := bigtable.tableValidatorProposals.ReadRows(ctx, ranges, func(r gcp_bigtable.Row) bool { + for _, ri := range r[PROPOSALS_FAMILY] { + keySplit := strings.Split(r.Key(), ":") + + proposalSlot, err := strconv.ParseUint(keySplit[3], 10, 64) + if err != nil { + logger.Errorf("error parsing slot from row key %v: %v", r.Key(), err) + return false + } + proposalSlot = max_block_number - proposalSlot + inclusionSlot := max_block_number - uint64(r[PROPOSALS_FAMILY][0].Timestamp)/1000 - if len(res[validator]) > 0 && res[validator][len(res[validator])-1].Slot == proposalSlot { - res[validator][len(res[validator])-1].Slot = proposalSlot - res[validator][len(res[validator])-1].Status = status - } else { - res[validator] = append(res[validator], &types.ValidatorProposal{ - Index: validator, - Status: status, - Slot: proposalSlot, - }) - } + status := uint64(1) + if inclusionSlot == max_block_number { + inclusionSlot = 0 + status = 2 + } - } - return true - }, gcp_bigtable.RowFilter(filter)) - if err != nil { + validator, err := bigtable.validatorKeyToIndex(keySplit[1]) + if err != nil { + logger.Errorf("error parsing validator from column key %v: %v", ri.Column, err) + return false + } + + resMux.Lock() + if res[validator] == nil { + res[validator] = make([]*types.ValidatorProposal, 0) + } + + if len(res[validator]) > 0 && res[validator][len(res[validator])-1].Slot == proposalSlot { + res[validator][len(res[validator])-1].Slot = proposalSlot + res[validator][len(res[validator])-1].Status = status + } else { + res[validator] = append(res[validator], &types.ValidatorProposal{ + Index: validator, + Status: status, + Slot: proposalSlot, + }) + } + resMux.Unlock() + + } + return true + }, gcp_bigtable.RowFilter(filter)) + + return err + }) + } + + if err := g.Wait(); err != nil { return nil, err } @@ -1563,6 +1640,14 @@ func (bigtable *Bigtable) SaveValidatorIncomeDetails(epoch uint64, rewards map[u // GetValidatorIncomeDetailsHistory returns the validator income details // startEpoch & endEpoch are inclusive func (bigtable *Bigtable) GetValidatorIncomeDetailsHistory(validators []uint64, startEpoch uint64, endEpoch uint64) (map[uint64]map[uint64]*itypes.ValidatorEpochIncome, error) { + + if len(validators) == 0 { + return nil, fmt.Errorf("passing empty validator array is unsupported") + } + + batchSize := 1000 + concurrency := 10 + if startEpoch > endEpoch { startEpoch = 0 } @@ -1570,43 +1655,68 @@ func (bigtable *Bigtable) GetValidatorIncomeDetailsHistory(validators []uint64, ctx, cancel := context.WithTimeout(context.Background(), time.Second*180) defer cancel() - ranges := bigtable.getValidatorsEpochRanges(validators, startEpoch, endEpoch) res := make(map[uint64]map[uint64]*itypes.ValidatorEpochIncome, len(validators)) + resMux := &sync.Mutex{} filter := gcp_bigtable.LatestNFilter(1) - err := bigtable.tableValidatorIncomeDetails.ReadRows(ctx, ranges, func(r gcp_bigtable.Row) bool { - keySplit := strings.Split(r.Key(), ":") + g, gCtx := errgroup.WithContext(ctx) + g.SetLimit(concurrency) - validator, err := bigtable.validatorKeyToIndex(keySplit[1]) - if err != nil { - logger.Errorf("error parsing validator from row key %v: %v", r.Key(), err) - return false - } + for i := 0; i < len(validators); i += batchSize { - epoch, err := strconv.ParseUint(keySplit[2], 10, 64) - if err != nil { - logger.Errorf("error parsing epoch from row key %v: %v", r.Key(), err) - return false + upperBound := i + batchSize + if len(validators) < upperBound { + upperBound = len(validators) } + vals := validators[i:upperBound] - for _, ri := range r[INCOME_DETAILS_COLUMN_FAMILY] { - incomeDetails := &itypes.ValidatorEpochIncome{} - err = proto.Unmarshal(ri.Value, incomeDetails) - if err != nil { - logger.Errorf("error decoding validator income data for row %v: %v", r.Key(), err) - return false + g.Go(func() error { + select { + case <-gCtx.Done(): + return gCtx.Err() + default: } + ranges := bigtable.getValidatorsEpochRanges(vals, startEpoch, endEpoch) + err := bigtable.tableValidatorIncomeDetails.ReadRows(ctx, ranges, func(r gcp_bigtable.Row) bool { + keySplit := strings.Split(r.Key(), ":") - if res[validator] == nil { - res[validator] = make(map[uint64]*itypes.ValidatorEpochIncome) - } + validator, err := bigtable.validatorKeyToIndex(keySplit[1]) + if err != nil { + logger.Errorf("error parsing validator from row key %v: %v", r.Key(), err) + return false + } - res[validator][max_epoch-epoch] = incomeDetails - } - return true - }, gcp_bigtable.RowFilter(filter)) - if err != nil { + epoch, err := strconv.ParseUint(keySplit[2], 10, 64) + if err != nil { + logger.Errorf("error parsing epoch from row key %v: %v", r.Key(), err) + return false + } + + for _, ri := range r[INCOME_DETAILS_COLUMN_FAMILY] { + incomeDetails := &itypes.ValidatorEpochIncome{} + err = proto.Unmarshal(ri.Value, incomeDetails) + if err != nil { + logger.Errorf("error decoding validator income data for row %v: %v", r.Key(), err) + return false + } + + resMux.Lock() + if res[validator] == nil { + res[validator] = make(map[uint64]*itypes.ValidatorEpochIncome) + } + + res[validator][max_epoch-epoch] = incomeDetails + resMux.Unlock() + } + return true + }, gcp_bigtable.RowFilter(filter)) + + return err + }) + } + + if err := g.Wait(); err != nil { return nil, err } @@ -1620,9 +1730,6 @@ func (bigtable *Bigtable) GetAggregatedValidatorIncomeDetailsHistory(validators startEpoch = 0 } - ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(time.Minute*10)) - defer cancel() - type ResultContainer struct { mu sync.Mutex res map[uint64]*itypes.ValidatorEpochIncome @@ -1630,9 +1737,7 @@ func (bigtable *Bigtable) GetAggregatedValidatorIncomeDetailsHistory(validators resultContainer := ResultContainer{} resultContainer.res = make(map[uint64]*itypes.ValidatorEpochIncome, len(validators)) - g, gCtx := errgroup.WithContext(ctx) - batchSize := 100 - g.SetLimit(10) + batchSize := 10000 for i := 0; i < len(validators); i += batchSize { upperBound := i + batchSize @@ -1643,48 +1748,36 @@ func (bigtable *Bigtable) GetAggregatedValidatorIncomeDetailsHistory(validators logrus.Infof("retrieving validator income stats for validators %v - %v", vals[0], vals[len(vals)-1]) - g.Go(func() error { - select { - case <-gCtx.Done(): - return gCtx.Err() - default: - } - res, err := bigtable.GetValidatorIncomeDetailsHistory(vals, startEpoch, endEpoch) + res, err := bigtable.GetValidatorIncomeDetailsHistory(vals, startEpoch, endEpoch) - if err != nil { - return err - } - resultContainer.mu.Lock() - for validator, epochs := range res { - for _, rewardDetails := range epochs { - - if resultContainer.res[validator] == nil { - resultContainer.res[validator] = &itypes.ValidatorEpochIncome{} - } + if err != nil { + return nil, err + } + resultContainer.mu.Lock() + for validator, epochs := range res { + for _, rewardDetails := range epochs { - resultContainer.res[validator].AttestationHeadReward += rewardDetails.AttestationHeadReward - resultContainer.res[validator].AttestationSourceReward += rewardDetails.AttestationSourceReward - resultContainer.res[validator].AttestationSourcePenalty += rewardDetails.AttestationSourcePenalty - resultContainer.res[validator].AttestationTargetReward += rewardDetails.AttestationTargetReward - resultContainer.res[validator].AttestationTargetPenalty += rewardDetails.AttestationTargetPenalty - resultContainer.res[validator].FinalityDelayPenalty += rewardDetails.FinalityDelayPenalty - resultContainer.res[validator].ProposerSlashingInclusionReward += rewardDetails.ProposerSlashingInclusionReward - resultContainer.res[validator].ProposerAttestationInclusionReward += rewardDetails.ProposerAttestationInclusionReward - resultContainer.res[validator].ProposerSyncInclusionReward += rewardDetails.ProposerSyncInclusionReward - resultContainer.res[validator].SyncCommitteeReward += rewardDetails.SyncCommitteeReward - resultContainer.res[validator].SyncCommitteePenalty += rewardDetails.SyncCommitteePenalty - resultContainer.res[validator].SlashingReward += rewardDetails.SlashingReward - resultContainer.res[validator].SlashingPenalty += rewardDetails.SlashingPenalty - resultContainer.res[validator].TxFeeRewardWei = utils.AddBigInts(resultContainer.res[validator].TxFeeRewardWei, rewardDetails.TxFeeRewardWei) + if resultContainer.res[validator] == nil { + resultContainer.res[validator] = &itypes.ValidatorEpochIncome{} } - } - resultContainer.mu.Unlock() - return nil - }) - } - if err := g.Wait(); err != nil { - return nil, err + resultContainer.res[validator].AttestationHeadReward += rewardDetails.AttestationHeadReward + resultContainer.res[validator].AttestationSourceReward += rewardDetails.AttestationSourceReward + resultContainer.res[validator].AttestationSourcePenalty += rewardDetails.AttestationSourcePenalty + resultContainer.res[validator].AttestationTargetReward += rewardDetails.AttestationTargetReward + resultContainer.res[validator].AttestationTargetPenalty += rewardDetails.AttestationTargetPenalty + resultContainer.res[validator].FinalityDelayPenalty += rewardDetails.FinalityDelayPenalty + resultContainer.res[validator].ProposerSlashingInclusionReward += rewardDetails.ProposerSlashingInclusionReward + resultContainer.res[validator].ProposerAttestationInclusionReward += rewardDetails.ProposerAttestationInclusionReward + resultContainer.res[validator].ProposerSyncInclusionReward += rewardDetails.ProposerSyncInclusionReward + resultContainer.res[validator].SyncCommitteeReward += rewardDetails.SyncCommitteeReward + resultContainer.res[validator].SyncCommitteePenalty += rewardDetails.SyncCommitteePenalty + resultContainer.res[validator].SlashingReward += rewardDetails.SlashingReward + resultContainer.res[validator].SlashingPenalty += rewardDetails.SlashingPenalty + resultContainer.res[validator].TxFeeRewardWei = utils.AddBigInts(resultContainer.res[validator].TxFeeRewardWei, rewardDetails.TxFeeRewardWei) + } + } + resultContainer.mu.Unlock() } return resultContainer.res, nil @@ -1692,129 +1785,10 @@ func (bigtable *Bigtable) GetAggregatedValidatorIncomeDetailsHistory(validators // Deletes all block data from bigtable func (bigtable *Bigtable) DeleteEpoch(epoch uint64) error { - - // First receive all keys that were written by this block (entities & indices) - keys := make([]string, 0, 33) - startSlot := epoch * utils.Config.Chain.Config.SlotsPerEpoch - endSlot := (epoch+1)*utils.Config.Chain.Config.SlotsPerEpoch - 1 - - logger.Infof("deleting epoch %v (slot %v to %v)", epoch, startSlot, endSlot) - for slot := startSlot; slot <= endSlot; slot++ { - keys = append(keys, fmt.Sprintf("%s:e:%s:s:%s", bigtable.chainId, reversedPaddedEpoch(slot/utils.Config.Chain.Config.SlotsPerEpoch), reversedPaddedSlot(slot))) - } - keys = append(keys, fmt.Sprintf("%s:e:b:%s", bigtable.chainId, reversedPaddedEpoch(epoch))) - - // for _, k := range keys { - // logger.Info(k) - // } - - // Delete all of those keys - mutsDelete := &types.BulkMutations{ - Keys: make([]string, 0, len(keys)), - Muts: make([]*gcp_bigtable.Mutation, 0, len(keys)), - } - for _, key := range keys { - mutDelete := gcp_bigtable.NewMutation() - mutDelete.DeleteRow() - mutsDelete.Keys = append(mutsDelete.Keys, key) - mutsDelete.Muts = append(mutsDelete.Muts, mutDelete) - } - - err := bigtable.WriteBulk(mutsDelete, bigtable.tableBeaconchain) - if err != nil { - return err - } - + logger.Fatal("NOT IMPLEMENTED") return nil } -func (bigtable *Bigtable) getSlotRanges(startEpoch uint64, endEpoch uint64) gcp_bigtable.RowRangeList { - - if endEpoch < startEpoch { // handle overflows - startEpoch = 0 - } - - ranges := gcp_bigtable.RowRangeList{} - if startEpoch == 0 { // special case when the 0 epoch is included - rangeEnd := fmt.Sprintf("%s:e:%s:s:%s", bigtable.chainId, reversedPaddedEpoch(0), ":") - rangeStart := fmt.Sprintf("%s:e:%s:s:", bigtable.chainId, reversedPaddedEpoch(0)) - ranges = append(ranges, gcp_bigtable.NewRange(rangeStart, rangeEnd)) - - // epochs are sorted descending, so start with the larges epoch and end with the smallest - // add ':', a character lexicographically after digits, to make the range inclusive - if startEpoch < endEpoch { - rangeEnd = fmt.Sprintf("%s:e:%s:s:%s", bigtable.chainId, reversedPaddedEpoch(startEpoch+1), ":") - rangeStart = fmt.Sprintf("%s:e:%s:s:", bigtable.chainId, reversedPaddedEpoch(endEpoch)) - ranges = append(ranges, gcp_bigtable.NewRange(rangeStart, rangeEnd)) - } - } else { - // epochs are sorted descending, so start with the larges epoch and end with the smallest - // add ':', a character lexicographically after digits, to make the range inclusive - rangeEnd := fmt.Sprintf("%s:e:%s:s:%s", bigtable.chainId, reversedPaddedEpoch(startEpoch), ":") - rangeStart := fmt.Sprintf("%s:e:%s:s:", bigtable.chainId, reversedPaddedEpoch(endEpoch)) - ranges = append(ranges, gcp_bigtable.NewRange(rangeStart, rangeEnd)) - } - return ranges -} - -func (bigtable *Bigtable) getEpochRanges(startEpoch uint64, endEpoch uint64) gcp_bigtable.RowRangeList { - - if endEpoch < startEpoch { // handle overflows - startEpoch = 0 - } - - ranges := gcp_bigtable.RowRangeList{} - if startEpoch == 0 { // special case when the 0 epoch is included - rangeEnd := fmt.Sprintf("%s:e:b:%s%s", bigtable.chainId, reversedPaddedEpoch(0), "\x00") - rangeStart := fmt.Sprintf("%s:e:b:%s", bigtable.chainId, reversedPaddedEpoch(0)) - ranges = append(ranges, gcp_bigtable.NewRange(rangeStart, rangeEnd)) - - // epochs are sorted descending, so start with the largest epoch and end with the smallest - // add \x00 to make the range inclusive - if startEpoch < endEpoch { - rangeEnd = fmt.Sprintf("%s:e:b:%s%s", bigtable.chainId, reversedPaddedEpoch(startEpoch+1), "\x00") - rangeStart = fmt.Sprintf("%s:e:b:%s", bigtable.chainId, reversedPaddedEpoch(endEpoch)) - ranges = append(ranges, gcp_bigtable.NewRange(rangeStart, rangeEnd)) - } - } else { - // epochs are sorted descending, so start with the largest epoch and end with the smallest - // add \x00 to make the range inclusive - rangeEnd := fmt.Sprintf("%s:e:b:%s%s", bigtable.chainId, reversedPaddedEpoch(startEpoch), "\x00") - rangeStart := fmt.Sprintf("%s:e:b:%s", bigtable.chainId, reversedPaddedEpoch(endEpoch)) - ranges = append(ranges, gcp_bigtable.NewRange(rangeStart, rangeEnd)) - } - return ranges -} - -func (bigtable *Bigtable) getValidatorEpochRanges(validatorIndex uint64, startEpoch uint64, endEpoch uint64) gcp_bigtable.RowRangeList { - - if endEpoch < startEpoch { // handle overflows - startEpoch = 0 - } - - ranges := gcp_bigtable.RowRangeList{} - if startEpoch == 0 { // special case when the 0 epoch is included - rangeEnd := fmt.Sprintf("%s:%s:%s%s", bigtable.chainId, bigtable.validatorIndexToKey(validatorIndex), reversedPaddedEpoch(0), "\x00") - rangeStart := fmt.Sprintf("%s:%s:%s", bigtable.chainId, bigtable.validatorIndexToKey(validatorIndex), reversedPaddedEpoch(0)) - ranges = append(ranges, gcp_bigtable.NewRange(rangeStart, rangeEnd)) - - // epochs are sorted descending, so start with the largest epoch and end with the smallest - // add \x00 to make the range inclusive - if startEpoch < endEpoch { - rangeEnd = fmt.Sprintf("%s:%s:%s%s", bigtable.chainId, bigtable.validatorIndexToKey(validatorIndex), reversedPaddedEpoch(startEpoch+1), "\x00") - rangeStart = fmt.Sprintf("%s:%s:%s", bigtable.chainId, bigtable.validatorIndexToKey(validatorIndex), reversedPaddedEpoch(endEpoch)) - ranges = append(ranges, gcp_bigtable.NewRange(rangeStart, rangeEnd)) - } - } else { - // epochs are sorted descending, so start with the largest epoch and end with the smallest - // add \x00 to make the range inclusive - rangeEnd := fmt.Sprintf("%s:%s:%s%s", bigtable.chainId, bigtable.validatorIndexToKey(validatorIndex), reversedPaddedEpoch(startEpoch), "\x00") - rangeStart := fmt.Sprintf("%s:%s:%s", bigtable.chainId, bigtable.validatorIndexToKey(validatorIndex), reversedPaddedEpoch(endEpoch)) - ranges = append(ranges, gcp_bigtable.NewRange(rangeStart, rangeEnd)) - } - return ranges -} - func (bigtable *Bigtable) getValidatorsEpochRanges(validatorIndices []uint64, startEpoch uint64, endEpoch uint64) gcp_bigtable.RowRangeList { if endEpoch < startEpoch { // handle overflows diff --git a/db/statistics.go b/db/statistics.go index 9064da0287..3b736b4172 100644 --- a/db/statistics.go +++ b/db/statistics.go @@ -1130,9 +1130,7 @@ func WriteValidatorFailedAttestationsStatisticsForDay(validators []uint64, day u // first key is the batch start index and the second is the validator id validatorMap := map[uint64]*types.ValidatorMissedAttestationsStatistic{} mux := sync.Mutex{} - g, gCtx := errgroup.WithContext(ctx) - batchSize := 100 - g.SetLimit(10) + batchSize := 10000 for i := 0; i < len(validators); i += batchSize { upperBound := i + batchSize @@ -1142,27 +1140,16 @@ func WriteValidatorFailedAttestationsStatisticsForDay(validators []uint64, day u vals := validators[i:upperBound] logrus.Infof("retrieving validator missed attestations stats for validators %v - %v", vals[0], vals[len(vals)-1]) - g.Go(func() error { - select { - case <-gCtx.Done(): - return gCtx.Err() - default: - } - ma, err := BigtableClient.GetValidatorMissedAttestationsCount(vals, firstEpoch, lastEpoch) - if err != nil { - return fmt.Errorf("error in GetValidatorMissedAttestationsCount for fromEpoch [%v] and toEpoch [%v]: %w", firstEpoch, lastEpoch, err) - } - mux.Lock() - for validator, stats := range ma { - validatorMap[validator] = stats - } - mux.Unlock() - return nil - }) - } - if err := g.Wait(); err != nil { - return err + ma, err := BigtableClient.GetValidatorMissedAttestationsCount(vals, firstEpoch, lastEpoch) + if err != nil { + return fmt.Errorf("error in GetValidatorMissedAttestationsCount for fromEpoch [%v] and toEpoch [%v]: %w", firstEpoch, lastEpoch, err) + } + mux.Lock() + for validator, stats := range ma { + validatorMap[validator] = stats + } + mux.Unlock() } logrus.Infof("fetching 'failed attestations' done in %v, now we export them to the db", time.Since(start)) @@ -1173,8 +1160,8 @@ func WriteValidatorFailedAttestationsStatisticsForDay(validators []uint64, day u maArr = append(maArr, stat) } - g, gCtx = errgroup.WithContext(ctx) - + g, gCtx := errgroup.WithContext(ctx) + g.SetLimit(50) dbBatchSize := 100 // max: 65535 / 4, but we are faster with smaller batches for b := 0; b < len(maArr); b += dbBatchSize { From 3309967132b196662c450f7021f147d5a3514155 Mon Sep 17 00:00:00 2001 From: peter <1674920+peterbitfly@users.noreply.github.com> Date: Thu, 24 Aug 2023 10:14:35 +0200 Subject: [PATCH 003/106] (BIDS-2379) add bigtable schema migration function --- db/bigtable.go | 317 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 317 insertions(+) diff --git a/db/bigtable.go b/db/bigtable.go index 5f9deb1de2..fd8da0befa 100644 --- a/db/bigtable.go +++ b/db/bigtable.go @@ -1637,6 +1637,323 @@ func (bigtable *Bigtable) SaveValidatorIncomeDetails(epoch uint64, rewards map[u return nil } +func (bigtable *Bigtable) MigrateEpochSchemaV1ToV2(epoch uint64) error { + funcStart := time.Now() + + defer func() { + logger.Infof("migration of epoch %v completed in %v", epoch, time.Since(funcStart)) + }() + + // start := time.Now() + + type validatorEpochData struct { + ValidatorIndex uint64 + Proposals map[uint64]uint64 + AttestationTargetSlot uint64 + AttestationInclusionSlot uint64 + SyncParticipation map[uint64]uint64 + EffectiveBalance uint64 + Balance uint64 + IncomeDetails *itypes.ValidatorEpochIncome + } + + epochData := make(map[uint64]*validatorEpochData) + filter := gcp_bigtable.LatestNFilter(1) + ctx := context.Background() + + prefixEpochRange := gcp_bigtable.PrefixRange(fmt.Sprintf("%s:e:b:%s", bigtable.chainId, reversedPaddedEpoch(epoch))) + + err := bigtable.tableBeaconchain.ReadRows(ctx, prefixEpochRange, func(r gcp_bigtable.Row) bool { + // logger.Infof("processing row %v", r.Key()) + + keySplit := strings.Split(r.Key(), ":") + + rowKeyEpoch, err := strconv.ParseUint(keySplit[3], 10, 64) + if err != nil { + logger.Errorf("error parsing epoch from row key %v: %v", r.Key(), err) + return false + } + + rowKeyEpoch = max_epoch - rowKeyEpoch + + if epoch != rowKeyEpoch { + logger.Errorf("retrieved different epoch than requested, requested: %d, retrieved: %d", epoch, rowKeyEpoch) + } + + logger.Infof("epoch is %d", rowKeyEpoch) + + for columnFamily, readItems := range r { + + for _, ri := range readItems { + + if ri.Column == "stats:sum" { // skip migrating the total epoch income stats + continue + } + + validator, err := strconv.ParseUint(strings.TrimPrefix(ri.Column, columnFamily+":"), 10, 64) + if err != nil { + logger.Errorf("error parsing validator from column key %v: %v", ri.Column, err) + return false + } + + // logger.Infof("retrieved field %s from column family %s for validator %d", ri.Column, columnFamily, validator) + + if epochData[validator] == nil { + epochData[validator] = &validatorEpochData{ + ValidatorIndex: validator, + Proposals: make(map[uint64]uint64), + SyncParticipation: make(map[uint64]uint64), + } + } + + if columnFamily == VALIDATOR_BALANCES_FAMILY { + // logger.Infof("processing balance data for validator %d", validator) + balances := ri.Value + balanceBytes := balances[0:8] + effectiveBalanceBytes := balances[8:16] + epochData[validator].Balance = binary.LittleEndian.Uint64(balanceBytes) + epochData[validator].EffectiveBalance = binary.LittleEndian.Uint64(effectiveBalanceBytes) + } else if columnFamily == INCOME_DETAILS_COLUMN_FAMILY { + // logger.Infof("processing income details data for validator %d", validator) + incomeDetails := &itypes.ValidatorEpochIncome{} + err = proto.Unmarshal(ri.Value, incomeDetails) + if err != nil { + logger.Errorf("error decoding validator income data for row %v: %v", r.Key(), err) + return false + } + + epochData[validator].IncomeDetails = incomeDetails + } else { + logger.Errorf("retrieved unexpected column family %s", columnFamily) + } + } + } + + return true + }, gcp_bigtable.RowFilter(filter)) + + if err != nil { + return err + } + + // logger.Infof("retrieved epoch data for %d validators in %v", len(epochData), time.Since(start)) + // start = time.Now() + + prefixEpochSlotRange := gcp_bigtable.PrefixRange(fmt.Sprintf("%s:e:%s:s:", bigtable.chainId, reversedPaddedEpoch(epoch))) + + err = bigtable.tableBeaconchain.ReadRows(ctx, prefixEpochSlotRange, func(r gcp_bigtable.Row) bool { + // logger.Infof("processing row %v", r.Key()) + + keySplit := strings.Split(r.Key(), ":") + + rowKeyEpoch, err := strconv.ParseUint(keySplit[2], 10, 64) + if err != nil { + logger.Errorf("error parsing epoch from row key %v: %v", r.Key(), err) + return false + } + + rowKeyEpoch = max_epoch - rowKeyEpoch + + if epoch != rowKeyEpoch { + logger.Errorf("retrieved different epoch than requested, requested: %d, retrieved: %d", epoch, rowKeyEpoch) + } + + slot, err := strconv.ParseUint(keySplit[4], 10, 64) + if err != nil { + logger.Errorf("error parsing slot from row key %v: %v", r.Key(), err) + return false + } + slot = max_block_number - slot + + // logger.Infof("epoch is %d, slot is %d", rowKeyEpoch, slot) + + for columnFamily, readItems := range r { + + for _, ri := range readItems { + + validator, err := strconv.ParseUint(strings.TrimPrefix(ri.Column, columnFamily+":"), 10, 64) + if err != nil { + logger.Errorf("error parsing validator from column key %v: %v", ri.Column, err) + return false + } + + inclusionSlot := uint64(0) + + if ri.Timestamp > 0 { + inclusionSlot = max_block_number - uint64(ri.Timestamp)/1000 + } + + // logger.Infof("retrieved field %s from column family %s for validator %d", ri.Column, columnFamily, validator) + + if epochData[validator] == nil { + epochData[validator] = &validatorEpochData{ + ValidatorIndex: validator, + } + } + + if columnFamily == ATTESTATIONS_FAMILY { + // logger.Infof("processing balance data for validator %d", validator) + epochData[validator].AttestationTargetSlot = slot + epochData[validator].AttestationInclusionSlot = inclusionSlot + // logger.Infof("processing attestation data for validator %d, target slot %d, inclusion slot %d", validator, slot, inclusionSlot) + } else if columnFamily == PROPOSALS_FAMILY { + epochData[validator].Proposals[slot] = inclusionSlot + // logger.Infof("processing proposer data for validator %d, proposal slot %d, inclusion slot %d", validator, slot, inclusionSlot) + } else if columnFamily == SYNC_COMMITTEES_FAMILY { + epochData[validator].SyncParticipation[slot] = inclusionSlot + //logger.Infof("processing sync data for validator %d, proposal slot %d, inclusion slot %d", validator, slot, inclusionSlot) + } else { + logger.Errorf("retrieved unexpected column family %s", columnFamily) + } + } + } + + return true + }, gcp_bigtable.RowFilter(filter)) + + if err != nil { + return err + } + + // logger.Infof("retrieved slot data for %d validators in %v", len(epochData), time.Since(start)) + // start = time.Now() + + // save validator balance data + validators := make([]*types.Validator, 0, len(epochData)) + + for _, validator := range epochData { + validators = append(validators, &types.Validator{ + Index: validator.ValidatorIndex, + EffectiveBalance: validator.EffectiveBalance, + Balance: validator.Balance, + }) + } + + err = bigtable.SaveValidatorBalances(epoch, validators) + if err != nil { + return err + } + // logger.Infof("migrated balance data in %v", time.Since(start)) + // start = time.Now() + + i := 0 + mutsInclusionSlot := make([]*gcp_bigtable.Mutation, 0, 100000) + keysInclusionSlot := make([]string, 0, 100000) + + for _, validator := range epochData { + mutInclusionSlot := gcp_bigtable.NewMutation() + mutInclusionSlot.Set(ATTESTATIONS_FAMILY, fmt.Sprintf("%d", validator.AttestationTargetSlot), gcp_bigtable.Timestamp((max_block_number-validator.AttestationInclusionSlot)*1000), []byte{}) + key := fmt.Sprintf("%s:%s:%s", bigtable.chainId, bigtable.validatorIndexToKey(validator.ValidatorIndex), reversedPaddedEpoch(epoch)) + + mutsInclusionSlot = append(mutsInclusionSlot, mutInclusionSlot) + keysInclusionSlot = append(keysInclusionSlot, key) + + if i%100000 == 0 { + errs, err := bigtable.tableValidatorAttestations.ApplyBulk(ctx, keysInclusionSlot, mutsInclusionSlot) + if err != nil { + return err + } + for _, err := range errs { + return err + } + mutsInclusionSlot = make([]*gcp_bigtable.Mutation, 0, 100000) + keysInclusionSlot = make([]string, 0, 100000) + } + i++ + } + + if len(mutsInclusionSlot) > 0 { + errs, err := bigtable.tableValidatorAttestations.ApplyBulk(ctx, keysInclusionSlot, mutsInclusionSlot) + + if err != nil { + return err + } + + for _, err := range errs { + return err + } + } + // logger.Infof("migrated attestation data in %v", time.Since(start)) + // start = time.Now() + + mutsProposals := make([]*gcp_bigtable.Mutation, 0, 100000) + keysProposals := make([]string, 0, 100000) + + for _, validator := range epochData { + if len(validator.Proposals) == 0 { + continue + } + for slot, inclusionSlot := range validator.Proposals { + mut := gcp_bigtable.NewMutation() + mut.Set(PROPOSALS_FAMILY, "b", gcp_bigtable.Timestamp((max_block_number-inclusionSlot)*1000), []byte{}) + key := fmt.Sprintf("%s:%s:%s:%s", bigtable.chainId, bigtable.validatorIndexToKey(validator.ValidatorIndex), reversedPaddedEpoch(epoch), reversedPaddedSlot(slot)) + + mutsProposals = append(mutsProposals, mut) + keysProposals = append(keysProposals, key) + } + } + errs, err := bigtable.tableValidatorProposals.ApplyBulk(ctx, keysProposals, mutsProposals) + + if err != nil { + return err + } + + for _, err := range errs { + return err + } + // logger.Infof("migrated proposal data in %v", time.Since(start)) + // start = time.Now() + + mutsSync := make([]*gcp_bigtable.Mutation, 0, 100000) + keysSync := make([]string, 0, 100000) + + for _, validator := range epochData { + + if len(validator.SyncParticipation) == 0 { + continue + } + for slot, inclusionSlot := range validator.SyncParticipation { + mut := gcp_bigtable.NewMutation() + mut.Set(SYNC_COMMITTEES_FAMILY, "s", gcp_bigtable.Timestamp((max_block_number-inclusionSlot)*1000), []byte{}) + + key := fmt.Sprintf("%s:%s:%s:%s", bigtable.chainId, bigtable.validatorIndexToKey(validator.ValidatorIndex), reversedPaddedEpoch(epoch), reversedPaddedSlot(slot)) + mutsSync = append(mutsSync, mut) + keysSync = append(keysSync, key) + } + } + + errs, err = bigtable.tableValidatorSyncCommittees.ApplyBulk(ctx, keysSync, mutsSync) + + if err != nil { + return err + } + + for _, err := range errs { + return err + } + // logger.Infof("migrated sync data in %v", time.Since(start)) + // start = time.Now() + + incomeData := make(map[uint64]*itypes.ValidatorEpochIncome) + for _, validator := range epochData { + if validator.IncomeDetails == nil { + continue + } + incomeData[validator.ValidatorIndex] = validator.IncomeDetails + } + + err = bigtable.SaveValidatorIncomeDetails(epoch, incomeData) + if err != nil { + return err + } + + // logger.Infof("migrated income data in %v", time.Since(start)) + // start = time.Now() + + return nil + +} + // GetValidatorIncomeDetailsHistory returns the validator income details // startEpoch & endEpoch are inclusive func (bigtable *Bigtable) GetValidatorIncomeDetailsHistory(validators []uint64, startEpoch uint64, endEpoch uint64) (map[uint64]map[uint64]*itypes.ValidatorEpochIncome, error) { From 7478a913abde88aa72ea57b92409b04261835fd9 Mon Sep 17 00:00:00 2001 From: peter <1674920+peterbitfly@users.noreply.github.com> Date: Mon, 28 Aug 2023 10:41:45 +0200 Subject: [PATCH 004/106] (BIDS-2379) fix attestation slot parsing --- db/bigtable.go | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/db/bigtable.go b/db/bigtable.go index fd8da0befa..442e03ea91 100644 --- a/db/bigtable.go +++ b/db/bigtable.go @@ -399,10 +399,10 @@ func machineMetricRowParts(r string) (bool, uint64, string, string) { func (bigtable *Bigtable) SaveValidatorBalances(epoch uint64, validators []*types.Validator) error { - ctx, cancel := context.WithTimeout(context.Background(), time.Second*30) + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) defer cancel() - start := time.Now() + // start := time.Now() ts := gcp_bigtable.Timestamp(0) muts := make([]*gcp_bigtable.Mutation, 0, 100000) @@ -450,7 +450,7 @@ func (bigtable *Bigtable) SaveValidatorBalances(epoch uint64, validators []*type } } - logger.Infof("exported validator balances to bigtable in %v", time.Since(start)) + // logger.Infof("exported validator balances to bigtable in %v", time.Since(start)) return nil } @@ -1128,7 +1128,8 @@ func (bigtable *Bigtable) GetValidatorMissedAttestationHistory(validators []uint } for _, ri := range r[ATTESTATIONS_FAMILY] { - attesterSlot, err := strconv.ParseUint(ri.Column, 10, 64) + attesterSlotString := strings.Replace(ri.Column, ATTESTATIONS_FAMILY+":", "", 1) + attesterSlot, err := strconv.ParseUint(attesterSlotString, 10, 64) if err != nil { logger.Errorf("error parsing slot from row key %v: %v", r.Key(), err) return false @@ -1563,7 +1564,7 @@ func (bigtable *Bigtable) GetValidatorProposalHistory(validators []uint64, start } func (bigtable *Bigtable) SaveValidatorIncomeDetails(epoch uint64, rewards map[uint64]*itypes.ValidatorEpochIncome) error { - ctx, cancel := context.WithTimeout(context.Background(), time.Second*30) + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) defer cancel() start := time.Now() @@ -1680,7 +1681,7 @@ func (bigtable *Bigtable) MigrateEpochSchemaV1ToV2(epoch uint64) error { logger.Errorf("retrieved different epoch than requested, requested: %d, retrieved: %d", epoch, rowKeyEpoch) } - logger.Infof("epoch is %d", rowKeyEpoch) + // logger.Infof("epoch is %d", rowKeyEpoch) for columnFamily, readItems := range r { From 31ad6e73d04ceb2272cf3f065bcecd37ac4faff1 Mon Sep 17 00:00:00 2001 From: peter <1674920+peterbitfly@users.noreply.github.com> Date: Mon, 28 Aug 2023 11:03:10 +0200 Subject: [PATCH 005/106] (BIDS-2379) fix staticchheck --- db/bigtable.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/db/bigtable.go b/db/bigtable.go index 442e03ea91..95900b107d 100644 --- a/db/bigtable.go +++ b/db/bigtable.go @@ -2103,8 +2103,7 @@ func (bigtable *Bigtable) GetAggregatedValidatorIncomeDetailsHistory(validators // Deletes all block data from bigtable func (bigtable *Bigtable) DeleteEpoch(epoch uint64) error { - logger.Fatal("NOT IMPLEMENTED") - return nil + return fmt.Errorf("NOT IMPLEMENTED") } func (bigtable *Bigtable) getValidatorsEpochRanges(validatorIndices []uint64, startEpoch uint64, endEpoch uint64) gcp_bigtable.RowRangeList { From 4b3069cb6eb9ed771b57cb188998f1de087e37f6 Mon Sep 17 00:00:00 2001 From: peter <1674920+peterbitfly@users.noreply.github.com> Date: Mon, 28 Aug 2023 20:55:08 +0200 Subject: [PATCH 006/106] (BIDS-2379) update bigtable config --- bigtable_config.md | 30 +++++++++++++++++++++++++----- 1 file changed, 25 insertions(+), 5 deletions(-) diff --git a/bigtable_config.md b/bigtable_config.md index cc850bf59e..f4ab660183 100644 --- a/bigtable_config.md +++ b/bigtable_config.md @@ -2,15 +2,35 @@ This document summarized the bigtable configuration options and table definitions required to run the beaconcha.in explorer. All settings can be applied either by using the GCP bigtable web interface or the `cbt` tool. ---- -Table name: `beaconchain` +Table name: `beaconchain_validator_balances` + +Column families: +* Name: `vb` | GC Policy: None + +---- +Table name: `beaconchain_validator_attestations` + +Column families: +* Name: `at` | GC Policy: None + +---- +Table name: `beaconchain_validator_proposals` + +Column families: +* Name: `pr` | GC Policy: None + +---- +Table name: `beaconchain_validator_sync` + +Column families: +* Name: `sc` | GC Policy: None + +---- +Table name: `beaconchain_validator_income` Column families: -* Name: `at` | GC Policy: Version based policy with a maximum of 1 versions * Name: `id` | GC Policy: None -* Name: `pr` | GC Policy: Version based policy with a maximum of 1 versions -* Name: `sc` | GC Policy: Version based policy with a maximum of 1 versions * Name: `stats` | GC Policy: None -* Name: `vb` | GC Policy: None ---- Table name: `beaconchain_validators` From b3a618f9dd6d555ea48438278223fd8721bd35d1 Mon Sep 17 00:00:00 2001 From: peter <1674920+peterbitfly@users.noreply.github.com> Date: Mon, 28 Aug 2023 20:59:19 +0200 Subject: [PATCH 007/106] (BIDS-2379) update bigtable config --- bigtable_config.md | 98 +++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 97 insertions(+), 1 deletion(-) diff --git a/bigtable_config.md b/bigtable_config.md index f4ab660183..7c948bfb14 100644 --- a/bigtable_config.md +++ b/bigtable_config.md @@ -4,70 +4,149 @@ This document summarized the bigtable configuration options and table definition ---- Table name: `beaconchain_validator_balances` +``` +cbt -project $PROJECT -instance $INSTANCE createtable beaconchain_validator_balances +``` + Column families: * Name: `vb` | GC Policy: None +``` +cbt -project $PROJECT -instance $INSTANCE createfamily vb +``` + ---- Table name: `beaconchain_validator_attestations` +``` +cbt -project $PROJECT -instance $INSTANCE createtable beaconchain_validator_attestations +``` + Column families: * Name: `at` | GC Policy: None +``` +cbt -project $PROJECT -instance $INSTANCE createfamily at +``` ---- Table name: `beaconchain_validator_proposals` +``` +cbt -project $PROJECT -instance $INSTANCE createtable beaconchain_validator_proposals +``` + Column families: * Name: `pr` | GC Policy: None +``` +cbt -project $PROJECT -instance $INSTANCE createfamily pr +``` ---- Table name: `beaconchain_validator_sync` +``` +cbt -project $PROJECT -instance $INSTANCE createtable beaconchain_validator_sync +``` + Column families: * Name: `sc` | GC Policy: None +``` +cbt -project $PROJECT -instance $INSTANCE createfamily sc +``` ---- Table name: `beaconchain_validator_income` +``` +cbt -project $PROJECT -instance $INSTANCE createtable beaconchain_validator_income +``` + Column families: * Name: `id` | GC Policy: None * Name: `stats` | GC Policy: None +``` +cbt -project $PROJECT -instance $INSTANCE createfamily id +cbt -project $PROJECT -instance $INSTANCE createfamily stats +``` ---- Table name: `beaconchain_validators` +``` +cbt -project $PROJECT -instance $INSTANCE createtable beaconchain_validators +``` + Column families: * Name: `at` | GC Policy: Version based policy with a maximum of 1 versions +``` +cbt -project $PROJECT -instance $INSTANCE createfamily at +``` ---- Table name: `blocks` +``` +cbt -project $PROJECT -instance $INSTANCE createtable blocks +``` + Column families: * Name: `default` | GC Policy: Version based policy with a maximum of 1 versions +``` +cbt -project $PROJECT -instance $INSTANCE createfamily default +``` ---- Table name: `cache` +``` +cbt -project $PROJECT -instance $INSTANCE createtable cache +``` + Column families: * Name: `10_min` | GC Policy: Version based policy with a maximum of 1 versions and a maximum age of 10 minutes * Name: `1_day` | GC Policy: Version based policy with a maximum of 1 versions and a maximum age of 1 day * Name: `1_hour` | GC Policy: Version based policy with a maximum of 1 versions and a maximum age of 1 hour +``` +cbt -project $PROJECT -instance $INSTANCE createfamily 10_min +cbt -project $PROJECT -instance $INSTANCE createfamily 1_day +cbt -project $PROJECT -instance $INSTANCE createfamily 1_hour +``` ---- Table name: `data` +``` +cbt -project $PROJECT -instance $INSTANCE createtable data +``` + Column families: * Name: `c` | GC Policy: Age based policy with a max age of 1 day * Name: `f` | GC Policy: None +``` +cbt -project $PROJECT -instance $INSTANCE createfamily c +cbt -project $PROJECT -instance $INSTANCE createfamily f +``` ---- Table name: `machine_metrics` +``` +cbt -project $PROJECT -instance $INSTANCE createtable machine_metrics +``` + Column families: * Name: `mm` | GC Policy: Age based policy with a max age of 31 days +``` +cbt -project $PROJECT -instance $INSTANCE createfamily mm +``` ---- Table name: `metadata` +``` +cbt -project $PROJECT -instance $INSTANCE createtable metadata +``` + Column families: * Name: `a` | GC Policy: None * Name: `c` | GC Policy: None @@ -76,9 +155,26 @@ Column families: * Name: `erc721` | GC Policy: None * Name: `series` | GC Policy: Version based policy with a maximum of 1 versions +``` +cbt -project $PROJECT -instance $INSTANCE createfamily a +cbt -project $PROJECT -instance $INSTANCE createfamily c +cbt -project $PROJECT -instance $INSTANCE createfamily erc1155 +cbt -project $PROJECT -instance $INSTANCE createfamily erc20 +cbt -project $PROJECT -instance $INSTANCE createfamily erc721 +cbt -project $PROJECT -instance $INSTANCE createfamily series +``` ---- Table name: `metadata_updates` +``` +cbt -project $PROJECT -instance $INSTANCE createtable metadata_updates +``` + Column families: * Name: `blocks` | GC Policy: Age based policy with a max age of 1 day -* Name: `f` | GC Policy: None \ No newline at end of file +* Name: `f` | GC Policy: None + +``` +cbt -project $PROJECT -instance $INSTANCE createfamily blocks +cbt -project $PROJECT -instance $INSTANCE createfamily f +``` \ No newline at end of file From a33d3c079b59f9cfea15e2fede28e8637299690a Mon Sep 17 00:00:00 2001 From: peter <1674920+peterbitfly@users.noreply.github.com> Date: Mon, 28 Aug 2023 21:03:21 +0200 Subject: [PATCH 008/106] (BIDS-2379) update bigtable config --- bigtable_config.md | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/bigtable_config.md b/bigtable_config.md index 7c948bfb14..d16ddbe88e 100644 --- a/bigtable_config.md +++ b/bigtable_config.md @@ -81,6 +81,8 @@ Column families: ``` cbt -project $PROJECT -instance $INSTANCE createfamily at + +cbt -project $PROJECT -instance $INSTANCE setgcpolicy beaconchain_validators at maxversions=1 ``` ---- Table name: `blocks` @@ -94,6 +96,8 @@ Column families: ``` cbt -project $PROJECT -instance $INSTANCE createfamily default + +cbt -project $PROJECT -instance $INSTANCE setgcpolicy blocks default maxversions=1 ``` ---- Table name: `cache` @@ -111,6 +115,10 @@ Column families: cbt -project $PROJECT -instance $INSTANCE createfamily 10_min cbt -project $PROJECT -instance $INSTANCE createfamily 1_day cbt -project $PROJECT -instance $INSTANCE createfamily 1_hour + +cbt -project $PROJECT -instance $INSTANCE setgcpolicy cache 10_min maxage=10m and maxversions=1 +cbt -project $PROJECT -instance $INSTANCE setgcpolicy cache 1_day maxage=1d and maxversions=1 +cbt -project $PROJECT -instance $INSTANCE setgcpolicy cache 1_hour maxage=1h and maxversions=1 ``` ---- Table name: `data` @@ -126,6 +134,8 @@ Column families: ``` cbt -project $PROJECT -instance $INSTANCE createfamily c cbt -project $PROJECT -instance $INSTANCE createfamily f + +cbt -project $PROJECT -instance $INSTANCE setgcpolicy data c maxage=1d ``` ---- Table name: `machine_metrics` @@ -162,6 +172,8 @@ cbt -project $PROJECT -instance $INSTANCE createfamily erc1155 cbt -project $PROJECT -instance $INSTANCE createfamily erc20 cbt -project $PROJECT -instance $INSTANCE createfamily erc721 cbt -project $PROJECT -instance $INSTANCE createfamily series + +cbt -project $PROJECT -instance $INSTANCE setgcpolicy metadata series maxversions=1 ``` ---- Table name: `metadata_updates` @@ -177,4 +189,6 @@ Column families: ``` cbt -project $PROJECT -instance $INSTANCE createfamily blocks cbt -project $PROJECT -instance $INSTANCE createfamily f + +cbt -project $PROJECT -instance $INSTANCE setgcpolicy metadata_updates blocks maxage=1d ``` \ No newline at end of file From 6980ca6ba421c2fb9d1404bd479d6af7163f1d15 Mon Sep 17 00:00:00 2001 From: peter <1674920+peterbitfly@users.noreply.github.com> Date: Mon, 28 Aug 2023 21:18:29 +0200 Subject: [PATCH 009/106] (BIDS-2379) update bigtable schema --- bigtable_config.md | 46 ++++++++++++++++++++++++---------------------- 1 file changed, 24 insertions(+), 22 deletions(-) diff --git a/bigtable_config.md b/bigtable_config.md index d16ddbe88e..acbbf097ef 100644 --- a/bigtable_config.md +++ b/bigtable_config.md @@ -12,7 +12,7 @@ Column families: * Name: `vb` | GC Policy: None ``` -cbt -project $PROJECT -instance $INSTANCE createfamily vb +cbt -project $PROJECT -instance $INSTANCE createfamily beaconchain_validator_balances vb ``` ---- @@ -26,7 +26,7 @@ Column families: * Name: `at` | GC Policy: None ``` -cbt -project $PROJECT -instance $INSTANCE createfamily at +cbt -project $PROJECT -instance $INSTANCE createfamily beaconchain_validator_attestations at ``` ---- Table name: `beaconchain_validator_proposals` @@ -39,7 +39,7 @@ Column families: * Name: `pr` | GC Policy: None ``` -cbt -project $PROJECT -instance $INSTANCE createfamily pr +cbt -project $PROJECT -instance $INSTANCE createfamily beaconchain_validator_proposals pr ``` ---- Table name: `beaconchain_validator_sync` @@ -52,7 +52,7 @@ Column families: * Name: `sc` | GC Policy: None ``` -cbt -project $PROJECT -instance $INSTANCE createfamily sc +cbt -project $PROJECT -instance $INSTANCE createfamily beaconchain_validator_sync sc ``` ---- Table name: `beaconchain_validator_income` @@ -66,8 +66,8 @@ Column families: * Name: `stats` | GC Policy: None ``` -cbt -project $PROJECT -instance $INSTANCE createfamily id -cbt -project $PROJECT -instance $INSTANCE createfamily stats +cbt -project $PROJECT -instance $INSTANCE createfamily beaconchain_validator_income id +cbt -project $PROJECT -instance $INSTANCE createfamily beaconchain_validator_income stats ``` ---- Table name: `beaconchain_validators` @@ -80,7 +80,7 @@ Column families: * Name: `at` | GC Policy: Version based policy with a maximum of 1 versions ``` -cbt -project $PROJECT -instance $INSTANCE createfamily at +cbt -project $PROJECT -instance $INSTANCE createfamily beaconchain_validators at cbt -project $PROJECT -instance $INSTANCE setgcpolicy beaconchain_validators at maxversions=1 ``` @@ -95,7 +95,7 @@ Column families: * Name: `default` | GC Policy: Version based policy with a maximum of 1 versions ``` -cbt -project $PROJECT -instance $INSTANCE createfamily default +cbt -project $PROJECT -instance $INSTANCE createfamily blocks default cbt -project $PROJECT -instance $INSTANCE setgcpolicy blocks default maxversions=1 ``` @@ -112,9 +112,9 @@ Column families: * Name: `1_hour` | GC Policy: Version based policy with a maximum of 1 versions and a maximum age of 1 hour ``` -cbt -project $PROJECT -instance $INSTANCE createfamily 10_min -cbt -project $PROJECT -instance $INSTANCE createfamily 1_day -cbt -project $PROJECT -instance $INSTANCE createfamily 1_hour +cbt -project $PROJECT -instance $INSTANCE createfamily cache 10_min +cbt -project $PROJECT -instance $INSTANCE createfamily cache 1_day +cbt -project $PROJECT -instance $INSTANCE createfamily cache 1_hour cbt -project $PROJECT -instance $INSTANCE setgcpolicy cache 10_min maxage=10m and maxversions=1 cbt -project $PROJECT -instance $INSTANCE setgcpolicy cache 1_day maxage=1d and maxversions=1 @@ -132,8 +132,8 @@ Column families: * Name: `f` | GC Policy: None ``` -cbt -project $PROJECT -instance $INSTANCE createfamily c -cbt -project $PROJECT -instance $INSTANCE createfamily f +cbt -project $PROJECT -instance $INSTANCE createfamily data c +cbt -project $PROJECT -instance $INSTANCE createfamily data f cbt -project $PROJECT -instance $INSTANCE setgcpolicy data c maxage=1d ``` @@ -148,7 +148,9 @@ Column families: * Name: `mm` | GC Policy: Age based policy with a max age of 31 days ``` -cbt -project $PROJECT -instance $INSTANCE createfamily mm +cbt -project $PROJECT -instance $INSTANCE createfamily machine_metrics mm + +cbt -project $PROJECT -instance $INSTANCE setgcpolicy machine_metrics mm maxage=31d ``` ---- Table name: `metadata` @@ -166,12 +168,12 @@ Column families: * Name: `series` | GC Policy: Version based policy with a maximum of 1 versions ``` -cbt -project $PROJECT -instance $INSTANCE createfamily a -cbt -project $PROJECT -instance $INSTANCE createfamily c -cbt -project $PROJECT -instance $INSTANCE createfamily erc1155 -cbt -project $PROJECT -instance $INSTANCE createfamily erc20 -cbt -project $PROJECT -instance $INSTANCE createfamily erc721 -cbt -project $PROJECT -instance $INSTANCE createfamily series +cbt -project $PROJECT -instance $INSTANCE createfamily metadata a +cbt -project $PROJECT -instance $INSTANCE createfamily metadata c +cbt -project $PROJECT -instance $INSTANCE createfamily metadata erc1155 +cbt -project $PROJECT -instance $INSTANCE createfamily metadata erc20 +cbt -project $PROJECT -instance $INSTANCE createfamily metadata erc721 +cbt -project $PROJECT -instance $INSTANCE createfamily metadata series cbt -project $PROJECT -instance $INSTANCE setgcpolicy metadata series maxversions=1 ``` @@ -187,8 +189,8 @@ Column families: * Name: `f` | GC Policy: None ``` -cbt -project $PROJECT -instance $INSTANCE createfamily blocks -cbt -project $PROJECT -instance $INSTANCE createfamily f +cbt -project $PROJECT -instance $INSTANCE createfamily metadata_updates blocks +cbt -project $PROJECT -instance $INSTANCE createfamily metadata_updates f cbt -project $PROJECT -instance $INSTANCE setgcpolicy metadata_updates blocks maxage=1d ``` \ No newline at end of file From 344a8b2f66eac3ac454a01374b3909d378025f9d Mon Sep 17 00:00:00 2001 From: peter <1674920+peterbitfly@users.noreply.github.com> Date: Wed, 30 Aug 2023 11:16:32 +0200 Subject: [PATCH 010/106] (BIDS-2379) increase timeouts --- db/bigtable.go | 105 +++++++++++++++++++++++------- exporter/sync_committees_count.go | 2 +- 2 files changed, 83 insertions(+), 24 deletions(-) diff --git a/db/bigtable.go b/db/bigtable.go index 95900b107d..459ce4d10a 100644 --- a/db/bigtable.go +++ b/db/bigtable.go @@ -399,7 +399,7 @@ func machineMetricRowParts(r string) (bool, uint64, string, string) { func (bigtable *Bigtable) SaveValidatorBalances(epoch uint64, validators []*types.Validator) error { - ctx, cancel := context.WithTimeout(context.Background(), time.Minute) + ctx, cancel := context.WithTimeout(context.Background(), time.Minute*2) defer cancel() // start := time.Now() @@ -456,7 +456,7 @@ func (bigtable *Bigtable) SaveValidatorBalances(epoch uint64, validators []*type func (bigtable *Bigtable) SaveAttestationAssignments(epoch uint64, assignments map[string]uint64) error { - ctx, cancel := context.WithTimeout(context.Background(), time.Second*30) + ctx, cancel := context.WithTimeout(context.Background(), time.Minute*2) defer cancel() start := time.Now() @@ -477,9 +477,10 @@ func (bigtable *Bigtable) SaveAttestationAssignments(epoch uint64, assignments m validatorsPerSlot[attesterslot] = append(validatorsPerSlot[attesterslot], validator) } + muts := make([]*gcp_bigtable.Mutation, 0, 100000) + keys := make([]string, 0, 100000) + for slot, validators := range validatorsPerSlot { - muts := make([]*gcp_bigtable.Mutation, 0, 100000) - keys := make([]string, 0, 100000) for _, validator := range validators { mut := gcp_bigtable.NewMutation() @@ -489,6 +490,23 @@ func (bigtable *Bigtable) SaveAttestationAssignments(epoch uint64, assignments m muts = append(muts, mut) keys = append(keys, key) } + + if len(muts)%100000 == 0 { + errs, err := bigtable.tableValidatorAttestations.ApplyBulk(ctx, keys, muts) + + if err != nil { + return err + } + + for _, err := range errs { + return err + } + muts = make([]*gcp_bigtable.Mutation, 0, 100000) + keys = make([]string, 0, 100000) + } + } + + if len(muts) > 0 { errs, err := bigtable.tableValidatorAttestations.ApplyBulk(ctx, keys, muts) if err != nil { @@ -540,7 +558,9 @@ func (bigtable *Bigtable) SaveProposalAssignments(epoch uint64, assignments map[ func (bigtable *Bigtable) SaveSyncCommitteesAssignments(startSlot, endSlot uint64, validators []uint64) error { - ctx, cancel := context.WithTimeout(context.Background(), time.Minute*5) + return nil //disabled as not needed + + ctx, cancel := context.WithTimeout(context.Background(), time.Minute*10) defer cancel() start := time.Now() @@ -558,22 +578,41 @@ func (bigtable *Bigtable) SaveSyncCommitteesAssignments(startSlot, endSlot uint6 muts = append(muts, mut) keys = append(keys, key) - } - } + if len(muts)%100000 == 0 { + logger.Infof("saving %v mutations for sync duties", len(muts)) + errs, err := bigtable.tableValidatorSyncCommittees.ApplyBulk(ctx, keys, muts) - logger.Infof("saving %v mutations for sync duties", len(muts)) + if err != nil { + return err + } - errs, err := bigtable.tableValidatorSyncCommittees.ApplyBulk(ctx, keys, muts) + for _, err := range errs { + if err != nil { + return err + } + } + + muts = make([]*gcp_bigtable.Mutation, 0, 100000) + keys = make([]string, 0, 100000) + } + } - if err != nil { - return err } - for _, err := range errs { + if len(muts) > 0 { + logger.Infof("saving %v mutations for sync duties", len(muts)) + errs, err := bigtable.tableValidatorSyncCommittees.ApplyBulk(ctx, keys, muts) + if err != nil { return err } + + for _, err := range errs { + if err != nil { + return err + } + } } logger.Infof("exported sync committee assignments to bigtable in %v", time.Since(start)) @@ -598,7 +637,7 @@ func (bigtable *Bigtable) SaveAttestations(blocks map[uint64]map[string]*types.B } bigtable.lastAttestationCacheMux.Unlock() - ctx, cancel := context.WithTimeout(context.Background(), time.Second*30) + ctx, cancel := context.WithTimeout(context.Background(), time.Minute*2) defer cancel() start := time.Now() @@ -632,9 +671,10 @@ func (bigtable *Bigtable) SaveAttestations(blocks map[uint64]map[string]*types.B } } + mutsInclusionSlot := make([]*gcp_bigtable.Mutation, 0, 100000) + keysInclusionSlot := make([]string, 0, 100000) + for attestedSlot, inclusions := range attestationsBySlot { - mutsInclusionSlot := make([]*gcp_bigtable.Mutation, 0, 100000) - keysInclusionSlot := make([]string, 0, 100000) mutLastAttestationSlot := gcp_bigtable.NewMutation() mutLastAttestationSlotSet := false @@ -653,9 +693,34 @@ func (bigtable *Bigtable) SaveAttestations(blocks map[uint64]map[string]*types.B bigtable.lastAttestationCache[validator] = attestedSlot mutLastAttestationSlotSet = true } + } bigtable.lastAttestationCacheMux.Unlock() + attstart := time.Now() + + if len(mutsInclusionSlot)%100000 == 0 { + errs, err := bigtable.tableValidatorAttestations.ApplyBulk(ctx, keysInclusionSlot, mutsInclusionSlot) + if err != nil { + return err + } + for _, err := range errs { + return err + } + logger.Infof("applied attestation mutations in %v", time.Since(attstart)) + mutsInclusionSlot = make([]*gcp_bigtable.Mutation, 0, 100000) + keysInclusionSlot = make([]string, 0, 100000) + } + if mutLastAttestationSlotSet { + err := bigtable.tableValidators.Apply(ctx, fmt.Sprintf("%s:lastAttestationSlot", bigtable.chainId), mutLastAttestationSlot) + if err != nil { + return err + } + } + } + + if len(mutsInclusionSlot) > 0 { + attstart := time.Now() errs, err := bigtable.tableValidatorAttestations.ApplyBulk(ctx, keysInclusionSlot, mutsInclusionSlot) if err != nil { return err @@ -663,13 +728,7 @@ func (bigtable *Bigtable) SaveAttestations(blocks map[uint64]map[string]*types.B for _, err := range errs { return err } - - if mutLastAttestationSlotSet { - err = bigtable.tableValidators.Apply(ctx, fmt.Sprintf("%s:lastAttestationSlot", bigtable.chainId), mutLastAttestationSlot) - if err != nil { - return err - } - } + logger.Infof("applied attestation mutations in %v", time.Since(attstart)) } logger.Infof("exported attestations (new) to bigtable in %v", time.Since(start)) @@ -738,7 +797,7 @@ func (bigtable *Bigtable) SaveProposals(blocks map[uint64]map[string]*types.Bloc func (bigtable *Bigtable) SaveSyncComitteeDuties(blocks map[uint64]map[string]*types.Block) error { - ctx, cancel := context.WithTimeout(context.Background(), time.Second*30) + ctx, cancel := context.WithTimeout(context.Background(), time.Minute*2) defer cancel() start := time.Now() diff --git a/exporter/sync_committees_count.go b/exporter/sync_committees_count.go index 145eb68dc1..b60ae34359 100644 --- a/exporter/sync_committees_count.go +++ b/exporter/sync_committees_count.go @@ -13,7 +13,7 @@ func syncCommitteesCountExporter() { for { err := exportSyncCommitteesCount() if err != nil { - logrus.WithFields(logrus.Fields{"error": err}).Errorf("error exporting sync_committees_count_per_validator") + logrus.WithFields(logrus.Fields{"error": err}).Warnf("error exporting sync_committees_count_per_validator") } time.Sleep(time.Second * 12) } From b3b12d0be0806adcc831bbdc8c37149172e1d0af Mon Sep 17 00:00:00 2001 From: peter <1674920+peterbitfly@users.noreply.github.com> Date: Wed, 30 Aug 2023 12:20:12 +0200 Subject: [PATCH 011/106] (BIDS-2379) promote exportSyncCommitteesCount error to error level --- exporter/sync_committees_count.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/exporter/sync_committees_count.go b/exporter/sync_committees_count.go index b60ae34359..145eb68dc1 100644 --- a/exporter/sync_committees_count.go +++ b/exporter/sync_committees_count.go @@ -13,7 +13,7 @@ func syncCommitteesCountExporter() { for { err := exportSyncCommitteesCount() if err != nil { - logrus.WithFields(logrus.Fields{"error": err}).Warnf("error exporting sync_committees_count_per_validator") + logrus.WithFields(logrus.Fields{"error": err}).Errorf("error exporting sync_committees_count_per_validator") } time.Sleep(time.Second * 12) } From 12a6acb2abdcfb06f12ac212b4912a48250ef11a Mon Sep 17 00:00:00 2001 From: peter <1674920+peterbitfly@users.noreply.github.com> Date: Wed, 30 Aug 2023 13:41:26 +0200 Subject: [PATCH 012/106] (BIDS-2379) add instructions for a local testnet deployment --- local-deployment/README.md | 99 +++++++++++++++++++++++++++++ local-deployment/docker-compose.yml | 24 +++++++ local-deployment/init-bigtable.sh | 56 ++++++++++++++++ 3 files changed, 179 insertions(+) create mode 100644 local-deployment/README.md create mode 100644 local-deployment/docker-compose.yml create mode 100644 local-deployment/init-bigtable.sh diff --git a/local-deployment/README.md b/local-deployment/README.md new file mode 100644 index 0000000000..51c3ed9550 --- /dev/null +++ b/local-deployment/README.md @@ -0,0 +1,99 @@ +This guide outlines how to deploy the explorer using a local lh-geth testnet. Utilized postgres, redis and little_bigtable as data storage + +# Install docker +``` +sudo apt update +sudo apt-get install ca-certificates curl gnupg +sudo install -m 0755 -d /etc/apt/keyrings +curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo gpg --dearmor -o /etc/apt/keyrings/docker.gpg +sudo chmod a+r /etc/apt/keyrings/docker.gpg +echo \ + "deb [arch="$(dpkg --print-architecture)" signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/ubuntu \ + "$(. /etc/os-release && echo "$VERSION_CODENAME")" stable" | \ + sudo tee /etc/apt/sources.list.d/docker.list > /dev/null +sudo apt-get update +sudo apt-get install docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin +sudo usermod -aG docker $USER +``` +# Install golang +``` +wget https://go.dev/dl/go1.20.7.linux-amd64.tar.gz +sudo rm -rf /usr/local/go && sudo tar -C /usr/local -xzf go1.20.7.linux-amd64.tar.gz +# Add export PATH=$PATH:/usr/local/go/bin to $HOME/.profile +``` +Create the testnet directory +``` +mkdir testnet +cd testnet +``` +# Clone the lh repo +``` +git clone https://github.com/sigp/lighthouse.git +cd lighthouse +``` +``` +# setup rust dev environment +curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh +source "$HOME/.cargo/env" +``` +# Install lh build deps +``` +sudo apt install -y git gcc g++ make cmake pkg-config llvm-dev libclang-dev clang protobuf-compiler +``` +# Build & install lighthouse +``` +make +``` +# Build & install lcli +``` +make install-lcli +``` +# Download and install geth & bootnode binary +``` +cd ~ +wget https://gethstore.blob.core.windows.net/builds/geth-alltools-linux-amd64-1.12.2-bed84606.tar.gz +tar -xf geth-alltools-linux-amd64-1.12.2-bed84606.tar.gz +sudo cp geth-alltools-linux-amd64-1.12.2-bed84606/geth /usr/bin/ +sudo cp geth-alltools-linux-amd64-1.12.2-bed84606/bootnode /usr/bin/ +``` +# Clone the explorer repository +``` +cd ~/ +git clone https://github.com/gobitfly/eth2-beaconchain-explorer.git +cd eth2-beaconchain-explorer +``` +# Build the explorer binaries +``` +make all +make misc +``` +# Clone the little_bigtable repository +``` +cd ~/ +git clone https://github.com/gobitfly/little_bigtable.git +cd little_bigtable +``` +# Build the little_bigtable binary +``` +make +``` +# Start postgres & redis +``` +cd ~/eth2-beaconchain-explorer/local-deployment/ +docker compose up -d +``` +Redis will be available on port 6379 and postgres on port 5432 (username `postgres`, password `pass`, db `db`) +# Start little_bigtable +``` +# in a new terminal +~/little_bigtable/build/little_bigtable -db-file ~/testnet/bigtable.db +``` +lbt will be available on http://127.0.0.1:9000 +# Initialize the bigtable schema +``` +bash ~/eth2-beaconchain-explorer/local-deployment/init-bigtable.sh +``` +# Initialize the db schema +``` +BIGTABLE_EMULATOR_HOST="127.0.0.1:9000" ~/eth2-beaconchain-explorer/bin/misc -config ~/eth2-beaconchain-explorer/local-deployment/config.yml -command applyDbSchema +``` diff --git a/local-deployment/docker-compose.yml b/local-deployment/docker-compose.yml new file mode 100644 index 0000000000..50cab77378 --- /dev/null +++ b/local-deployment/docker-compose.yml @@ -0,0 +1,24 @@ +version: '3' +services: + ################ dbs + redis: + image: redis:7 + restart: unless-stopped + volumes: + - ./docker-volumes/redis:/data + ports: + - 6379:6379 + postgres: + image: postgres:14 + restart: unless-stopped + environment: + - POSTGRES_PASSWORD=pass + - POSTGRES_USER=postgres + - POSTGRES_DB=db + - PGDATA=/var/lib/postgresql/data + volumes: + - ./docker-volumes/postgres:/var/lib/postgresql/data + - ./tables.sql:/docker-entrypoint-initdb.d/init.sql + ports: + - 5432:5432 + diff --git a/local-deployment/init-bigtable.sh b/local-deployment/init-bigtable.sh new file mode 100644 index 0000000000..824ddb8d73 --- /dev/null +++ b/local-deployment/init-bigtable.sh @@ -0,0 +1,56 @@ +#!/bin/bash + +PROJECT="explorer" +INSTANCE="explorer" +HOST="127.0.0.1:9000" + +BIGTABLE_EMULATOR_HOST=$HOST cbt --project $PROJECT --instance $INSTANCE createtable beaconchain_validator_balances +BIGTABLE_EMULATOR_HOST=$HOST cbt --project $PROJECT --instance $INSTANCE createfamily beaconchain_validator_balances vb +BIGTABLE_EMULATOR_HOST=$HOST cbt --project $PROJECT --instance $INSTANCE createtable beaconchain_validator_attestations +BIGTABLE_EMULATOR_HOST=$HOST cbt --project $PROJECT --instance $INSTANCE createfamily beaconchain_validator_attestations at +BIGTABLE_EMULATOR_HOST=$HOST cbt --project $PROJECT --instance $INSTANCE createtable beaconchain_validator_proposals +BIGTABLE_EMULATOR_HOST=$HOST cbt --project $PROJECT --instance $INSTANCE createfamily beaconchain_validator_proposals pr +BIGTABLE_EMULATOR_HOST=$HOST cbt --project $PROJECT --instance $INSTANCE createtable beaconchain_validator_sync +BIGTABLE_EMULATOR_HOST=$HOST cbt --project $PROJECT --instance $INSTANCE createfamily beaconchain_validator_sync sc +BIGTABLE_EMULATOR_HOST=$HOST cbt --project $PROJECT --instance $INSTANCE createtable beaconchain_validator_income +BIGTABLE_EMULATOR_HOST=$HOST cbt --project $PROJECT --instance $INSTANCE createfamily beaconchain_validator_income id +BIGTABLE_EMULATOR_HOST=$HOST cbt --project $PROJECT --instance $INSTANCE createfamily beaconchain_validator_income stats +BIGTABLE_EMULATOR_HOST=$HOST cbt --project $PROJECT --instance $INSTANCE createtable beaconchain_validators +BIGTABLE_EMULATOR_HOST=$HOST cbt --project $PROJECT --instance $INSTANCE createfamily beaconchain_validators at + +BIGTABLE_EMULATOR_HOST=$HOST cbt --project $PROJECT --instance $INSTANCE setgcpolicy beaconchain_validators at maxversions=1 +BIGTABLE_EMULATOR_HOST=$HOST cbt --project $PROJECT --instance $INSTANCE createtable blocks +BIGTABLE_EMULATOR_HOST=$HOST cbt --project $PROJECT --instance $INSTANCE createfamily blocks default + +BIGTABLE_EMULATOR_HOST=$HOST cbt --project $PROJECT --instance $INSTANCE setgcpolicy blocks default maxversions=1 +BIGTABLE_EMULATOR_HOST=$HOST cbt --project $PROJECT --instance $INSTANCE createtable cache +BIGTABLE_EMULATOR_HOST=$HOST cbt --project $PROJECT --instance $INSTANCE createfamily cache 10_min +BIGTABLE_EMULATOR_HOST=$HOST cbt --project $PROJECT --instance $INSTANCE createfamily cache 1_day +BIGTABLE_EMULATOR_HOST=$HOST cbt --project $PROJECT --instance $INSTANCE createfamily cache 1_hour + +BIGTABLE_EMULATOR_HOST=$HOST cbt --project $PROJECT --instance $INSTANCE setgcpolicy cache 10_min maxage=10m and maxversions=1 +BIGTABLE_EMULATOR_HOST=$HOST cbt --project $PROJECT --instance $INSTANCE setgcpolicy cache 1_day maxage=1d and maxversions=1 +BIGTABLE_EMULATOR_HOST=$HOST cbt --project $PROJECT --instance $INSTANCE setgcpolicy cache 1_hour maxage=1h and maxversions=1 +BIGTABLE_EMULATOR_HOST=$HOST cbt --project $PROJECT --instance $INSTANCE createtable data +BIGTABLE_EMULATOR_HOST=$HOST cbt --project $PROJECT --instance $INSTANCE createfamily data c +BIGTABLE_EMULATOR_HOST=$HOST cbt --project $PROJECT --instance $INSTANCE createfamily data f + +BIGTABLE_EMULATOR_HOST=$HOST cbt --project $PROJECT --instance $INSTANCE setgcpolicy data c maxage=1d +BIGTABLE_EMULATOR_HOST=$HOST cbt --project $PROJECT --instance $INSTANCE createtable machine_metrics +BIGTABLE_EMULATOR_HOST=$HOST cbt --project $PROJECT --instance $INSTANCE createfamily machine_metrics mm + +BIGTABLE_EMULATOR_HOST=$HOST cbt --project $PROJECT --instance $INSTANCE setgcpolicy machine_metrics mm maxage=31d +BIGTABLE_EMULATOR_HOST=$HOST cbt --project $PROJECT --instance $INSTANCE createtable metadata +BIGTABLE_EMULATOR_HOST=$HOST cbt --project $PROJECT --instance $INSTANCE createfamily metadata a +BIGTABLE_EMULATOR_HOST=$HOST cbt --project $PROJECT --instance $INSTANCE createfamily metadata c +BIGTABLE_EMULATOR_HOST=$HOST cbt --project $PROJECT --instance $INSTANCE createfamily metadata erc1155 +BIGTABLE_EMULATOR_HOST=$HOST cbt --project $PROJECT --instance $INSTANCE createfamily metadata erc20 +BIGTABLE_EMULATOR_HOST=$HOST cbt --project $PROJECT --instance $INSTANCE createfamily metadata erc721 +BIGTABLE_EMULATOR_HOST=$HOST cbt --project $PROJECT --instance $INSTANCE createfamily metadata series + +BIGTABLE_EMULATOR_HOST=$HOST cbt --project $PROJECT --instance $INSTANCE setgcpolicy metadata series maxversions=1 +BIGTABLE_EMULATOR_HOST=$HOST cbt --project $PROJECT --instance $INSTANCE createtable metadata_updates +BIGTABLE_EMULATOR_HOST=$HOST cbt --project $PROJECT --instance $INSTANCE createfamily metadata_updates blocks +BIGTABLE_EMULATOR_HOST=$HOST cbt --project $PROJECT --instance $INSTANCE createfamily metadata_updates f + +BIGTABLE_EMULATOR_HOST=$HOST cbt --project $PROJECT --instance $INSTANCE setgcpolicy metadata_updates blocks maxage=1d \ No newline at end of file From 1496ddc92358834e5cb745be48e636bca602513c Mon Sep 17 00:00:00 2001 From: peter <1674920+peterbitfly@users.noreply.github.com> Date: Wed, 30 Aug 2023 14:04:16 +0200 Subject: [PATCH 013/106] (BIDS-2379) add config --- local-deployment/README.md | 11 ++++- local-deployment/testnet-config.yml | 64 +++++++++++++++++++++++++++++ 2 files changed, 74 insertions(+), 1 deletion(-) create mode 100644 local-deployment/testnet-config.yml diff --git a/local-deployment/README.md b/local-deployment/README.md index 51c3ed9550..153cc9d2df 100644 --- a/local-deployment/README.md +++ b/local-deployment/README.md @@ -19,13 +19,22 @@ sudo usermod -aG docker $USER ``` wget https://go.dev/dl/go1.20.7.linux-amd64.tar.gz sudo rm -rf /usr/local/go && sudo tar -C /usr/local -xzf go1.20.7.linux-amd64.tar.gz -# Add export PATH=$PATH:/usr/local/go/bin to $HOME/.profile +``` +Add the golang binaries to the path by adding the following lines to your ~/.profile file and then logout & login again +``` +export PATH=$PATH:/usr/local/go/bin +export PATH=$PATH:$HOME/go/bin ``` Create the testnet directory ``` mkdir testnet cd testnet ``` +# Install the cbt tool +``` +sudo apt remove google-cloud-cli +sudo apt install google-cloud-sdk-cbt +``` # Clone the lh repo ``` git clone https://github.com/sigp/lighthouse.git diff --git a/local-deployment/testnet-config.yml b/local-deployment/testnet-config.yml new file mode 100644 index 0000000000..c3452aaf6c --- /dev/null +++ b/local-deployment/testnet-config.yml @@ -0,0 +1,64 @@ +chain: + name: local-testnet + configPath: '~/' +readerDatabase: + name: db + host: 127.0.0.1 + port: "5432" + user: postgres + password: "pass" +writerDatabase: + name: db + host: 127.0.0.1 + port: "5432" + user: postgres + password: "pass" +bigtable: + project: explorer + instance: explorer +eth1ErigonEndpoint: 'http://127.0.0.1:8545' +eth1GethEndpoint: 'http://127.0.0.1:8545' +redisCacheEndpoint: '127.0.0.1:6379' +tieredCacheProvider: 'redis' +#tieredCacheProvider: 'bigtable' +frontend: + siteDomain: "local-testnet.beaconcha.in" + siteName: 'Open Source Ethereum (ETH) Testnet Explorer' # Name of the site, displayed in the title tag + siteSubtitle: "Showing a local testnet." + server: + host: '0.0.0.0' # Address to listen on + port: '8080' # Port to listen on + readerDatabase: + name: db + host: 127.0.0.1 + port: "5432" + user: postgres + password: "pass" + writerDatabase: + name: db + host: 27.0.0.1 + port: "15432" + user: postgres + password: "pass" + sessionSecret: "11111111111111111111111111111111" + jwtSigningSecret: "1111111111111111111111111111111111111111111111111111111111111111" + jwtIssuer: "localhost" + jwtValidityInMinutes: 30 + maxMailsPerEmailPerDay: 10 + mail: + mailgun: + sender: no-reply@localhost + domain: mg.localhost + privateKey: "key-11111111111111111111111111111111" + csrfAuthKey: '1111111111111111111111111111111111111111111111111111111111111111' +indexer: + enabled: true + # fullIndexOnStartup: false # Perform a one time full db index on startup + # indexMissingEpochsOnStartup: true # Check for missing epochs and export them after startup + node: + host: 127.0.0.1 + port: '4000' + type: lighthouse + eth1Endpoint: "http://127.0.0.1:8545" + eth1DepositContractAddress: '0x4242424242424242424242424242424242424242' + eth1DepositContractFirstBlock: 0 \ No newline at end of file From 80c1a311d727c8a713d95889b8a570ee102646d0 Mon Sep 17 00:00:00 2001 From: peter <1674920+peterbitfly@users.noreply.github.com> Date: Wed, 30 Aug 2023 14:09:44 +0200 Subject: [PATCH 014/106] (BIDS-2379) fix config file --- local-deployment/testnet-config.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/local-deployment/testnet-config.yml b/local-deployment/testnet-config.yml index c3452aaf6c..608a6cc2bf 100644 --- a/local-deployment/testnet-config.yml +++ b/local-deployment/testnet-config.yml @@ -1,6 +1,6 @@ chain: name: local-testnet - configPath: '~/' + configPath: './testnet/chain.yml' readerDatabase: name: db host: 127.0.0.1 @@ -36,7 +36,7 @@ frontend: password: "pass" writerDatabase: name: db - host: 27.0.0.1 + host: 127.0.0.1 port: "15432" user: postgres password: "pass" From cf8908c04d9bf386b08e7b47be71e087bd7df020 Mon Sep 17 00:00:00 2001 From: peter <1674920+peterbitfly@users.noreply.github.com> Date: Wed, 30 Aug 2023 14:10:28 +0200 Subject: [PATCH 015/106] (BIDS-2379) fix config --- local-deployment/testnet-config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/local-deployment/testnet-config.yml b/local-deployment/testnet-config.yml index 608a6cc2bf..f79f6ecaee 100644 --- a/local-deployment/testnet-config.yml +++ b/local-deployment/testnet-config.yml @@ -37,7 +37,7 @@ frontend: writerDatabase: name: db host: 127.0.0.1 - port: "15432" + port: "5432" user: postgres password: "pass" sessionSecret: "11111111111111111111111111111111" From 41cda79c7bf44fb48a2dee2400674ba89916e4cc Mon Sep 17 00:00:00 2001 From: peter <1674920+peterbitfly@users.noreply.github.com> Date: Wed, 30 Aug 2023 14:20:52 +0200 Subject: [PATCH 016/106] (BIDS-2379) update testnet config --- local-deployment/README.md | 14 ++++++++++++-- local-deployment/testnet-config.yml | 2 +- 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/local-deployment/README.md b/local-deployment/README.md index 153cc9d2df..1931c0b247 100644 --- a/local-deployment/README.md +++ b/local-deployment/README.md @@ -47,7 +47,7 @@ source "$HOME/.cargo/env" ``` # Install lh build deps ``` -sudo apt install -y git gcc g++ make cmake pkg-config llvm-dev libclang-dev clang protobuf-compiler +sudo apt install -y git gcc g++ make cmake pkg-config llvm-dev libclang-dev clang protobuf-compiler jq ``` # Build & install lighthouse ``` @@ -102,7 +102,17 @@ lbt will be available on http://127.0.0.1:9000 ``` bash ~/eth2-beaconchain-explorer/local-deployment/init-bigtable.sh ``` +# Start up the local testnet nodes +## Switch to the lighthous scripts directory +``` +cd testnet/lighthouse/scripts/local_testnet/ +``` +## Start the local testnet +``` +cd testnet/lighthouse/scripts/local_testnet/ +./start_local_testnet.sh -v 2 genesis.json +``` # Initialize the db schema ``` -BIGTABLE_EMULATOR_HOST="127.0.0.1:9000" ~/eth2-beaconchain-explorer/bin/misc -config ~/eth2-beaconchain-explorer/local-deployment/config.yml -command applyDbSchema +BIGTABLE_EMULATOR_HOST="127.0.0.1:9000" ~/eth2-beaconchain-explorer/bin/misc -config ~/eth2-beaconchain-explorer/local-deployment/testnet-config.yml -command applyDbSchema ``` diff --git a/local-deployment/testnet-config.yml b/local-deployment/testnet-config.yml index f79f6ecaee..92bd0e31c1 100644 --- a/local-deployment/testnet-config.yml +++ b/local-deployment/testnet-config.yml @@ -1,6 +1,6 @@ chain: name: local-testnet - configPath: './testnet/chain.yml' + configPath: './.lighthouse/local-testnet/testnet/config.yaml' readerDatabase: name: db host: 127.0.0.1 From d2e71fe5d4586d535e18c09f128b08fea00de222 Mon Sep 17 00:00:00 2001 From: peter <1674920+peterbitfly@users.noreply.github.com> Date: Wed, 30 Aug 2023 14:29:14 +0200 Subject: [PATCH 017/106] (BIDS-2379) update config --- local-deployment/README.md | 4 ++++ local-deployment/testnet-config.yml | 8 ++++---- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/local-deployment/README.md b/local-deployment/README.md index 1931c0b247..170003073a 100644 --- a/local-deployment/README.md +++ b/local-deployment/README.md @@ -116,3 +116,7 @@ cd testnet/lighthouse/scripts/local_testnet/ ``` BIGTABLE_EMULATOR_HOST="127.0.0.1:9000" ~/eth2-beaconchain-explorer/bin/misc -config ~/eth2-beaconchain-explorer/local-deployment/testnet-config.yml -command applyDbSchema ``` +# Start the indexer +``` +BIGTABLE_EMULATOR_HOST="127.0.0.1:9000" ~/eth2-beaconchain-explorer/bin/explorer -config ~/eth2-beaconchain-explorer/local-deployment/testnet-config.yml +``` diff --git a/local-deployment/testnet-config.yml b/local-deployment/testnet-config.yml index 92bd0e31c1..2a77f1703c 100644 --- a/local-deployment/testnet-config.yml +++ b/local-deployment/testnet-config.yml @@ -16,8 +16,8 @@ writerDatabase: bigtable: project: explorer instance: explorer -eth1ErigonEndpoint: 'http://127.0.0.1:8545' -eth1GethEndpoint: 'http://127.0.0.1:8545' +eth1ErigonEndpoint: 'http://127.0.0.1:5001' +eth1GethEndpoint: 'http://127.0.0.1:5001' redisCacheEndpoint: '127.0.0.1:6379' tieredCacheProvider: 'redis' #tieredCacheProvider: 'bigtable' @@ -57,8 +57,8 @@ indexer: # indexMissingEpochsOnStartup: true # Check for missing epochs and export them after startup node: host: 127.0.0.1 - port: '4000' + port: '8001' type: lighthouse - eth1Endpoint: "http://127.0.0.1:8545" + eth1Endpoint: "http://127.0.0.1:5001" eth1DepositContractAddress: '0x4242424242424242424242424242424242424242' eth1DepositContractFirstBlock: 0 \ No newline at end of file From 02739fa18b2bc4d305345caacfa985cbccb13e31 Mon Sep 17 00:00:00 2001 From: peter <1674920+peterbitfly@users.noreply.github.com> Date: Wed, 30 Aug 2023 14:30:07 +0200 Subject: [PATCH 018/106] (BIDS-2379) update config --- local-deployment/testnet-config.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/local-deployment/testnet-config.yml b/local-deployment/testnet-config.yml index 2a77f1703c..803263998d 100644 --- a/local-deployment/testnet-config.yml +++ b/local-deployment/testnet-config.yml @@ -16,8 +16,8 @@ writerDatabase: bigtable: project: explorer instance: explorer -eth1ErigonEndpoint: 'http://127.0.0.1:5001' -eth1GethEndpoint: 'http://127.0.0.1:5001' +eth1ErigonEndpoint: 'http://127.0.0.1:6001' +eth1GethEndpoint: 'http://127.0.0.1:6001' redisCacheEndpoint: '127.0.0.1:6379' tieredCacheProvider: 'redis' #tieredCacheProvider: 'bigtable' @@ -59,6 +59,6 @@ indexer: host: 127.0.0.1 port: '8001' type: lighthouse - eth1Endpoint: "http://127.0.0.1:5001" + eth1Endpoint: "http://127.0.0.1:6001" eth1DepositContractAddress: '0x4242424242424242424242424242424242424242' eth1DepositContractFirstBlock: 0 \ No newline at end of file From 9e6a82b173035100098a6ac87e5b2c6a1f8a5c58 Mon Sep 17 00:00:00 2001 From: peter <1674920+peterbitfly@users.noreply.github.com> Date: Wed, 30 Aug 2023 14:37:58 +0200 Subject: [PATCH 019/106] (BIDS-2379) do not crash on txpool_content error --- local-deployment/README.md | 4 ++++ services/services.go | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/local-deployment/README.md b/local-deployment/README.md index 170003073a..03bc258c62 100644 --- a/local-deployment/README.md +++ b/local-deployment/README.md @@ -120,3 +120,7 @@ BIGTABLE_EMULATOR_HOST="127.0.0.1:9000" ~/eth2-beaconchain-explorer/bin/misc -co ``` BIGTABLE_EMULATOR_HOST="127.0.0.1:9000" ~/eth2-beaconchain-explorer/bin/explorer -config ~/eth2-beaconchain-explorer/local-deployment/testnet-config.yml ``` +# Start the fdu +``` +BIGTABLE_EMULATOR_HOST="127.0.0.1:9000" ~/eth2-beaconchain-explorer/bin/frontend-data-updater -config ~/eth2-beaconchain-explorer/local-deployment/testnet-config.yml +``` diff --git a/services/services.go b/services/services.go index 40c779427f..e3a112670b 100644 --- a/services/services.go +++ b/services/services.go @@ -1319,7 +1319,7 @@ func getGasNowData() (*types.GasNowPageData, error) { err = client.Call(&raw, "txpool_content") if err != nil { - utils.LogFatal(err, "error getting raw json data from txpool_content", 0) + return nil, fmt.Errorf("error getting raw json data from txpool_content: %w", err) } txPoolContent := &TxPoolContent{} From 608a593ca208fb4ffc75dfeb34588ef0df7d0940 Mon Sep 17 00:00:00 2001 From: peter <1674920+peterbitfly@users.noreply.github.com> Date: Wed, 30 Aug 2023 14:42:43 +0200 Subject: [PATCH 020/106] (BIDS-2379) add frontend config --- local-deployment/testnet-config-frontend.yml | 65 ++++++++++++++++++++ 1 file changed, 65 insertions(+) create mode 100644 local-deployment/testnet-config-frontend.yml diff --git a/local-deployment/testnet-config-frontend.yml b/local-deployment/testnet-config-frontend.yml new file mode 100644 index 0000000000..417fc77aa6 --- /dev/null +++ b/local-deployment/testnet-config-frontend.yml @@ -0,0 +1,65 @@ +chain: + name: local-testnet + configPath: './.lighthouse/local-testnet/testnet/config.yaml' +readerDatabase: + name: db + host: 127.0.0.1 + port: "5432" + user: postgres + password: "pass" +writerDatabase: + name: db + host: 127.0.0.1 + port: "5432" + user: postgres + password: "pass" +bigtable: + project: explorer + instance: explorer +eth1ErigonEndpoint: 'http://127.0.0.1:6001' +eth1GethEndpoint: 'http://127.0.0.1:6001' +redisCacheEndpoint: '127.0.0.1:6379' +tieredCacheProvider: 'redis' +#tieredCacheProvider: 'bigtable' +frontend: + enabled: true + siteDomain: "local-testnet.beaconcha.in" + siteName: 'Open Source Ethereum (ETH) Testnet Explorer' # Name of the site, displayed in the title tag + siteSubtitle: "Showing a local testnet." + server: + host: '0.0.0.0' # Address to listen on + port: '8080' # Port to listen on + readerDatabase: + name: db + host: 127.0.0.1 + port: "5432" + user: postgres + password: "pass" + writerDatabase: + name: db + host: 127.0.0.1 + port: "5432" + user: postgres + password: "pass" + sessionSecret: "11111111111111111111111111111111" + jwtSigningSecret: "1111111111111111111111111111111111111111111111111111111111111111" + jwtIssuer: "localhost" + jwtValidityInMinutes: 30 + maxMailsPerEmailPerDay: 10 + mail: + mailgun: + sender: no-reply@localhost + domain: mg.localhost + privateKey: "key-11111111111111111111111111111111" + csrfAuthKey: '1111111111111111111111111111111111111111111111111111111111111111' +indexer: + enabled: false + # fullIndexOnStartup: false # Perform a one time full db index on startup + # indexMissingEpochsOnStartup: true # Check for missing epochs and export them after startup + node: + host: 127.0.0.1 + port: '8001' + type: lighthouse + eth1Endpoint: "http://127.0.0.1:6001" + eth1DepositContractAddress: '0x4242424242424242424242424242424242424242' + eth1DepositContractFirstBlock: 0 \ No newline at end of file From 0361765d1158aaeea9a42ea196615ed656bd5c10 Mon Sep 17 00:00:00 2001 From: peter <1674920+peterbitfly@users.noreply.github.com> Date: Wed, 30 Aug 2023 14:50:09 +0200 Subject: [PATCH 021/106] (BIDS-2379) update config --- local-deployment/testnet-config.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/local-deployment/testnet-config.yml b/local-deployment/testnet-config.yml index 803263998d..724be94ab2 100644 --- a/local-deployment/testnet-config.yml +++ b/local-deployment/testnet-config.yml @@ -1,6 +1,7 @@ chain: name: local-testnet configPath: './.lighthouse/local-testnet/testnet/config.yaml' + genesisTimestamp: 1693397704 readerDatabase: name: db host: 127.0.0.1 From 0ed588a38905b44c1dc929d9b89138e775e27f22 Mon Sep 17 00:00:00 2001 From: peter <1674920+peterbitfly@users.noreply.github.com> Date: Wed, 30 Aug 2023 14:53:51 +0200 Subject: [PATCH 022/106] (BIDS-2379) update frontend config --- local-deployment/testnet-config-frontend.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/local-deployment/testnet-config-frontend.yml b/local-deployment/testnet-config-frontend.yml index 417fc77aa6..12c26b8899 100644 --- a/local-deployment/testnet-config-frontend.yml +++ b/local-deployment/testnet-config-frontend.yml @@ -1,6 +1,7 @@ chain: name: local-testnet configPath: './.lighthouse/local-testnet/testnet/config.yaml' + genesisTimestamp: 1693397704 readerDatabase: name: db host: 127.0.0.1 From 36ce5c3283a81703d0aeb822914696d84366eb65 Mon Sep 17 00:00:00 2001 From: peter <1674920+peterbitfly@users.noreply.github.com> Date: Wed, 30 Aug 2023 15:38:00 +0200 Subject: [PATCH 023/106] (BIDS-2379) add utility to convert lh testnet config stub to full config --- cmd/misc/main.go | 169 ++++++++++++++++++++++++++++++++++++++++++----- types/config.go | 34 ++++++++++ 2 files changed, 186 insertions(+), 17 deletions(-) diff --git a/cmd/misc/main.go b/cmd/misc/main.go index 5f0ae4866e..318cd94f4c 100644 --- a/cmd/misc/main.go +++ b/cmd/misc/main.go @@ -3,6 +3,7 @@ package main import ( "bytes" "context" + "eth2-exporter/config" "eth2-exporter/db" "eth2-exporter/exporter" "eth2-exporter/rpc" @@ -12,6 +13,7 @@ import ( "eth2-exporter/version" "fmt" "math/big" + "os" "strconv" "strings" "time" @@ -20,6 +22,7 @@ import ( _ "github.com/jackc/pgx/v4/stdlib" utilMath "github.com/protolambda/zrnt/eth2/util/math" "golang.org/x/sync/errgroup" + "gopkg.in/yaml.v3" "flag" @@ -27,27 +30,28 @@ import ( ) var opts = struct { - Command string - User uint64 - TargetVersion int64 - StartEpoch uint64 - EndEpoch uint64 - StartDay uint64 - EndDay uint64 - Validator uint64 - StartBlock uint64 - EndBlock uint64 - BatchSize uint64 - DataConcurrency uint64 - Transformers string - Family string - Key string - DryRun bool + Command string + User uint64 + TargetVersion int64 + StartEpoch uint64 + EndEpoch uint64 + StartDay uint64 + EndDay uint64 + Validator uint64 + StartBlock uint64 + EndBlock uint64 + BatchSize uint64 + DataConcurrency uint64 + Transformers string + Family string + Key string + DryRun bool + FullConfigOutputPath string }{} func main() { configPath := flag.String("config", "config/default.config.yml", "Path to the config file") - flag.StringVar(&opts.Command, "command", "", "command to run, available: updateAPIKey, applyDbSchema, epoch-export, debug-rewards, clear-bigtable, index-old-eth1-blocks, update-aggregation-bits, historic-prices-export, index-missing-blocks, export-epoch-missed-slots, migrate-last-attestation-slot-bigtable") + flag.StringVar(&opts.Command, "command", "", "command to run, available: updateAPIKey, applyDbSchema, epoch-export, debug-rewards, clear-bigtable, index-old-eth1-blocks, update-aggregation-bits, historic-prices-export, index-missing-blocks, export-epoch-missed-slots, migrate-last-attestation-slot-bigtable, generate-config-from-testnet-stub") flag.Uint64Var(&opts.StartEpoch, "start-epoch", 0, "start epoch") flag.Uint64Var(&opts.EndEpoch, "end-epoch", 0, "end epoch") flag.Uint64Var(&opts.User, "user", 0, "user id") @@ -62,6 +66,7 @@ func main() { flag.Uint64Var(&opts.DataConcurrency, "data.concurrency", 30, "Concurrency to use when indexing data from bigtable") flag.Uint64Var(&opts.BatchSize, "data.batchSize", 1000, "Batch size") flag.StringVar(&opts.Transformers, "transformers", "", "Comma separated list of transformers used by the eth1 indexer") + flag.StringVar(&opts.FullConfigOutputPath, "output-path", "", "Path to write the full config to (for generate-config-from-testnet-stub command)") dryRun := flag.String("dry-run", "true", "if 'false' it deletes all rows starting with the key, per default it only logs the rows that would be deleted, but does not really delete them") versionFlag := flag.Bool("version", false, "Show version and exit") flag.Parse() @@ -71,6 +76,11 @@ func main() { return } + if opts.Command == "generate-config-from-testnet-stub" { + generateConfigFromTestnetStub(*configPath, opts.FullConfigOutputPath) + return + } + opts.DryRun = *dryRun != "false" logrus.WithField("config", *configPath).WithField("version", version.Version).Printf("starting") @@ -649,3 +659,128 @@ func exportHistoricPrices(dayStart uint64, dayEnd uint64) { logrus.Info("historic price update run completed") } + +func generateConfigFromTestnetStub(configPath string, fullConfigOutputPath string) { + + data, err := os.ReadFile(configPath) + + if err != nil { + logrus.Fatalf("error reading config file: %v", err) + } + + stub := &types.LocalTestnetConfigStub{} + + err = yaml.Unmarshal(data, &stub) + if err != nil { + logrus.Fatalf("error parsing config stub: %v", err) + } + + if stub.PRESETBASE == "mainnet" { + cfg := &types.ChainConfig{} + + err := yaml.Unmarshal([]byte(config.MainnetChainYml), cfg) + + if err != nil { + logrus.Fatalf("error initializing full config: %v", err) + } + + cfg.TerminalTotalDifficulty = stub.TERMINALTOTALDIFFICULTY + cfg.TerminalBlockHash = stub.TERMINALBLOCKHASH + cfg.TerminalBlockHashActivationEpoch, err = strconv.ParseUint(stub.TERMINALBLOCKHASHACTIVATIONEPOCH, 10, 64) + if err != nil { + logrus.Fatalf("error overwriting ALTAIRFORKEPOCH: %v", err) + } + cfg.MinGenesisActiveValidatorCount, err = strconv.ParseUint(stub.MINGENESISACTIVEVALIDATORCOUNT, 10, 64) + if err != nil { + logrus.Fatalf("error overwriting ALTAIRFORKEPOCH: %v", err) + } + cfg.MinGenesisTime, err = strconv.ParseInt(stub.MINGENESISTIME, 10, 64) + if err != nil { + logrus.Fatalf("error overwriting ALTAIRFORKEPOCH: %v", err) + } + cfg.GenesisForkVersion = stub.GENESISFORKVERSION + cfg.GenesisDelay, err = strconv.ParseUint(stub.GENESISDELAY, 10, 64) + if err != nil { + logrus.Fatalf("error overwriting ALTAIRFORKEPOCH: %v", err) + } + cfg.AltairForkVersion = stub.ALTAIRFORKVERSION + cfg.AltairForkEpoch, err = strconv.ParseUint(stub.ALTAIRFORKEPOCH, 10, 64) + if err != nil { + logrus.Fatalf("error overwriting ALTAIRFORKEPOCH: %v", err) + } + cfg.BellatrixForkVersion = stub.BELLATRIXFORKEPOCH + cfg.BellatrixForkEpoch, err = strconv.ParseUint(stub.BELLATRIXFORKEPOCH, 10, 64) + if err != nil { + logrus.Fatalf("error overwriting ALTAIRFORKEPOCH: %v", err) + } + cfg.CappellaForkVersion = stub.CAPELLAFORKVERSION + cfg.CappellaForkEpoch, err = strconv.ParseUint(stub.CAPELLAFORKEPOCH, 10, 64) + if err != nil { + logrus.Fatalf("error overwriting ALTAIRFORKEPOCH: %v", err) + } + cfg.SecondsPerSlot, err = strconv.ParseUint(stub.SECONDSPERSLOT, 10, 64) + if err != nil { + logrus.Fatalf("error overwriting ALTAIRFORKEPOCH: %v", err) + } + cfg.SecondsPerEth1Block, err = strconv.ParseUint(stub.SECONDSPERETH1BLOCK, 10, 64) + if err != nil { + logrus.Fatalf("error overwriting ALTAIRFORKEPOCH: %v", err) + } + cfg.MinValidatorWithdrawabilityDelay, err = strconv.ParseUint(stub.MINVALIDATORWITHDRAWABILITYDELAY, 10, 64) + if err != nil { + logrus.Fatalf("error overwriting ALTAIRFORKEPOCH: %v", err) + } + cfg.ShardCommitteePeriod, err = strconv.ParseUint(stub.SHARDCOMMITTEEPERIOD, 10, 64) + if err != nil { + logrus.Fatalf("error overwriting ALTAIRFORKEPOCH: %v", err) + } + cfg.Eth1FollowDistance, err = strconv.ParseUint(stub.ETH1FOLLOWDISTANCE, 10, 64) + if err != nil { + logrus.Fatalf("error overwriting ALTAIRFORKEPOCH: %v", err) + } + cfg.InactivityScoreBias, err = strconv.ParseUint(stub.INACTIVITYSCOREBIAS, 10, 64) + if err != nil { + logrus.Fatalf("error overwriting ALTAIRFORKEPOCH: %v", err) + } + cfg.InactivityScoreRecoveryRate, err = strconv.ParseUint(stub.INACTIVITYSCORERECOVERYRATE, 10, 64) + if err != nil { + logrus.Fatalf("error overwriting ALTAIRFORKEPOCH: %v", err) + } + cfg.EjectionBalance, err = strconv.ParseUint(stub.EJECTIONBALANCE, 10, 64) + if err != nil { + logrus.Fatalf("error overwriting ALTAIRFORKEPOCH: %v", err) + } + cfg.MinPerEpochChurnLimit, err = strconv.ParseUint(stub.MINPEREPOCHCHURNLIMIT, 10, 64) + if err != nil { + logrus.Fatalf("error overwriting ALTAIRFORKEPOCH: %v", err) + } + cfg.ChurnLimitQuotient, err = strconv.ParseUint(stub.CHURNLIMITQUOTIENT, 10, 64) + if err != nil { + logrus.Fatalf("error overwriting ALTAIRFORKEPOCH: %v", err) + } + cfg.ProposerScoreBoost, err = strconv.ParseUint(stub.PROPOSERSCOREBOOST, 10, 64) + if err != nil { + logrus.Fatalf("error overwriting ALTAIRFORKEPOCH: %v", err) + } + cfg.DepositChainID, err = strconv.ParseUint(stub.DEPOSITCHAINID, 10, 64) + if err != nil { + logrus.Fatalf("error overwriting ALTAIRFORKEPOCH: %v", err) + } + cfg.DepositNetworkID, err = strconv.ParseUint(stub.DEPOSITNETWORKID, 10, 64) + if err != nil { + logrus.Fatalf("error overwriting ALTAIRFORKEPOCH: %v", err) + } + cfg.DepositContractAddress = stub.DEPOSITCONTRACTADDRESS + + encodedConfig, err := yaml.Marshal(cfg) + if err != nil { + logrus.Fatalf("error encoding new chain config: %v", err) + } + + err = os.WriteFile(fullConfigOutputPath, encodedConfig, 0664) + if err != nil { + logrus.Fatalf("error writing new chain config: %v", err) + } + } + +} diff --git a/types/config.go b/types/config.go index 67eb98e40d..a44d57bfe0 100644 --- a/types/config.go +++ b/types/config.go @@ -216,3 +216,37 @@ type ServiceMonitoringConfiguration struct { Name string `yaml:"name" envconfig:"NAME"` Duration time.Duration `yaml:"duration" envconfig:"DURATION"` } + +type LocalTestnetConfigStub struct { + CONFIGNAME string `yaml:"CONFIG_NAME"` + PRESETBASE string `yaml:"PRESET_BASE"` + TERMINALTOTALDIFFICULTY string `yaml:"TERMINAL_TOTAL_DIFFICULTY"` + TERMINALBLOCKHASH string `yaml:"TERMINAL_BLOCK_HASH"` + TERMINALBLOCKHASHACTIVATIONEPOCH string `yaml:"TERMINAL_BLOCK_HASH_ACTIVATION_EPOCH"` + SAFESLOTSTOIMPORTOPTIMISTICALLY string `yaml:"SAFE_SLOTS_TO_IMPORT_OPTIMISTICALLY"` + MINGENESISACTIVEVALIDATORCOUNT string `yaml:"MIN_GENESIS_ACTIVE_VALIDATOR_COUNT"` + MINGENESISTIME string `yaml:"MIN_GENESIS_TIME"` + GENESISFORKVERSION string `yaml:"GENESIS_FORK_VERSION"` + GENESISDELAY string `yaml:"GENESIS_DELAY"` + ALTAIRFORKVERSION string `yaml:"ALTAIR_FORK_VERSION"` + ALTAIRFORKEPOCH string `yaml:"ALTAIR_FORK_EPOCH"` + BELLATRIXFORKVERSION string `yaml:"BELLATRIX_FORK_VERSION"` + BELLATRIXFORKEPOCH string `yaml:"BELLATRIX_FORK_EPOCH"` + CAPELLAFORKVERSION string `yaml:"CAPELLA_FORK_VERSION"` + CAPELLAFORKEPOCH string `yaml:"CAPELLA_FORK_EPOCH"` + SECONDSPERSLOT string `yaml:"SECONDS_PER_SLOT"` + SECONDSPERETH1BLOCK string `yaml:"SECONDS_PER_ETH1_BLOCK"` + MINVALIDATORWITHDRAWABILITYDELAY string `yaml:"MIN_VALIDATOR_WITHDRAWABILITY_DELAY"` + SHARDCOMMITTEEPERIOD string `yaml:"SHARD_COMMITTEE_PERIOD"` + ETH1FOLLOWDISTANCE string `yaml:"ETH1_FOLLOW_DISTANCE"` + SUBNETSPERNODE string `yaml:"SUBNETS_PER_NODE"` + INACTIVITYSCOREBIAS string `yaml:"INACTIVITY_SCORE_BIAS"` + INACTIVITYSCORERECOVERYRATE string `yaml:"INACTIVITY_SCORE_RECOVERY_RATE"` + EJECTIONBALANCE string `yaml:"EJECTION_BALANCE"` + MINPEREPOCHCHURNLIMIT string `yaml:"MIN_PER_EPOCH_CHURN_LIMIT"` + CHURNLIMITQUOTIENT string `yaml:"CHURN_LIMIT_QUOTIENT"` + PROPOSERSCOREBOOST string `yaml:"PROPOSER_SCORE_BOOST"` + DEPOSITCHAINID string `yaml:"DEPOSIT_CHAIN_ID"` + DEPOSITNETWORKID string `yaml:"DEPOSIT_NETWORK_ID"` + DEPOSITCONTRACTADDRESS string `yaml:"DEPOSIT_CONTRACT_ADDRESS"` +} From 9b863ed59968fdde39c4a8f0c8c0deb850aa35b8 Mon Sep 17 00:00:00 2001 From: peter <1674920+peterbitfly@users.noreply.github.com> Date: Wed, 30 Aug 2023 15:46:23 +0200 Subject: [PATCH 024/106] (BIDS-2379) update default config path --- local-deployment/testnet-config-frontend.yml | 2 +- local-deployment/testnet-config.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/local-deployment/testnet-config-frontend.yml b/local-deployment/testnet-config-frontend.yml index 12c26b8899..f3d660f08c 100644 --- a/local-deployment/testnet-config-frontend.yml +++ b/local-deployment/testnet-config-frontend.yml @@ -1,6 +1,6 @@ chain: name: local-testnet - configPath: './.lighthouse/local-testnet/testnet/config.yaml' + configPath: '.lighthouse/local-testnet/testnet/config.full.yaml' genesisTimestamp: 1693397704 readerDatabase: name: db diff --git a/local-deployment/testnet-config.yml b/local-deployment/testnet-config.yml index 724be94ab2..5326152bf3 100644 --- a/local-deployment/testnet-config.yml +++ b/local-deployment/testnet-config.yml @@ -1,6 +1,6 @@ chain: name: local-testnet - configPath: './.lighthouse/local-testnet/testnet/config.yaml' + configPath: '.lighthouse/local-testnet/testnet/config.full.yaml' genesisTimestamp: 1693397704 readerDatabase: name: db From a28e1885de6eadd6768d78faaef0dcb41f5579db Mon Sep 17 00:00:00 2001 From: peter <1674920+peterbitfly@users.noreply.github.com> Date: Wed, 30 Aug 2023 15:58:42 +0200 Subject: [PATCH 025/106] (BIDS-2379) update local guide --- local-deployment/README.md | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/local-deployment/README.md b/local-deployment/README.md index 03bc258c62..e4bcd69143 100644 --- a/local-deployment/README.md +++ b/local-deployment/README.md @@ -112,15 +112,23 @@ cd testnet/lighthouse/scripts/local_testnet/ cd testnet/lighthouse/scripts/local_testnet/ ./start_local_testnet.sh -v 2 genesis.json ``` +## Generate the explorer config from the lh testnet config stub +``` +eth2-beaconchain-explorer/bin/misc -command generate-config-from-testnet-stub -config .lighthouse/local-testnet/testnet/config.yaml -output-path .lighthouse/local-testnet/testnet/config.full.yaml +``` # Initialize the db schema ``` BIGTABLE_EMULATOR_HOST="127.0.0.1:9000" ~/eth2-beaconchain-explorer/bin/misc -config ~/eth2-beaconchain-explorer/local-deployment/testnet-config.yml -command applyDbSchema ``` # Start the indexer ``` -BIGTABLE_EMULATOR_HOST="127.0.0.1:9000" ~/eth2-beaconchain-explorer/bin/explorer -config ~/eth2-beaconchain-explorer/local-deployment/testnet-config.yml +CHAIN_GENESIS_TIMESTAMP= BIGTABLE_EMULATOR_HOST="127.0.0.1:9000" ~/eth2-beaconchain-explorer/bin/explorer -config ~/eth2-beaconchain-explorer/local-deployment/testnet-config.yml ``` # Start the fdu ``` -BIGTABLE_EMULATOR_HOST="127.0.0.1:9000" ~/eth2-beaconchain-explorer/bin/frontend-data-updater -config ~/eth2-beaconchain-explorer/local-deployment/testnet-config.yml +CHAIN_GENESIS_TIMESTAMP= BIGTABLE_EMULATOR_HOST="127.0.0.1:9000" ~/eth2-beaconchain-explorer/bin/frontend-data-updater -config ~/eth2-beaconchain-explorer/local-deployment/testnet-config.yml +``` +# Start the frontend +``` +CHAIN_GENESIS_TIMESTAMP= BIGTABLE_EMULATOR_HOST="127.0.0.1:9000" ~/eth2-beaconchain-explorer/bin/frontend-data-updater -config ~/eth2-beaconchain-explorer/local-deployment/testnet-config-frontend.yml ``` From 9e46c8c65bcc460b6054718cb8357c3b8ac2edb3 Mon Sep 17 00:00:00 2001 From: peter <1674920+peterbitfly@users.noreply.github.com> Date: Thu, 31 Aug 2023 08:50:41 +0200 Subject: [PATCH 026/106] (BIDS-2379) improve local deplyoment setup --- cmd/misc/main.go | 167 ++----------------- go.mod | 1 + go.sum | 2 + local-deployment/README.md | 60 ++----- local-deployment/testnet-config-frontend.yml | 2 - local-deployment/testnet-config.yml | 4 +- types/config.go | 130 +++++++++++---- utils/utils.go | 132 +++++++++++++++ 8 files changed, 263 insertions(+), 235 deletions(-) diff --git a/cmd/misc/main.go b/cmd/misc/main.go index 318cd94f4c..5eddb5d7bf 100644 --- a/cmd/misc/main.go +++ b/cmd/misc/main.go @@ -3,7 +3,6 @@ package main import ( "bytes" "context" - "eth2-exporter/config" "eth2-exporter/db" "eth2-exporter/exporter" "eth2-exporter/rpc" @@ -13,7 +12,6 @@ import ( "eth2-exporter/version" "fmt" "math/big" - "os" "strconv" "strings" "time" @@ -22,7 +20,6 @@ import ( _ "github.com/jackc/pgx/v4/stdlib" utilMath "github.com/protolambda/zrnt/eth2/util/math" "golang.org/x/sync/errgroup" - "gopkg.in/yaml.v3" "flag" @@ -30,23 +27,22 @@ import ( ) var opts = struct { - Command string - User uint64 - TargetVersion int64 - StartEpoch uint64 - EndEpoch uint64 - StartDay uint64 - EndDay uint64 - Validator uint64 - StartBlock uint64 - EndBlock uint64 - BatchSize uint64 - DataConcurrency uint64 - Transformers string - Family string - Key string - DryRun bool - FullConfigOutputPath string + Command string + User uint64 + TargetVersion int64 + StartEpoch uint64 + EndEpoch uint64 + StartDay uint64 + EndDay uint64 + Validator uint64 + StartBlock uint64 + EndBlock uint64 + BatchSize uint64 + DataConcurrency uint64 + Transformers string + Family string + Key string + DryRun bool }{} func main() { @@ -66,7 +62,6 @@ func main() { flag.Uint64Var(&opts.DataConcurrency, "data.concurrency", 30, "Concurrency to use when indexing data from bigtable") flag.Uint64Var(&opts.BatchSize, "data.batchSize", 1000, "Batch size") flag.StringVar(&opts.Transformers, "transformers", "", "Comma separated list of transformers used by the eth1 indexer") - flag.StringVar(&opts.FullConfigOutputPath, "output-path", "", "Path to write the full config to (for generate-config-from-testnet-stub command)") dryRun := flag.String("dry-run", "true", "if 'false' it deletes all rows starting with the key, per default it only logs the rows that would be deleted, but does not really delete them") versionFlag := flag.Bool("version", false, "Show version and exit") flag.Parse() @@ -76,11 +71,6 @@ func main() { return } - if opts.Command == "generate-config-from-testnet-stub" { - generateConfigFromTestnetStub(*configPath, opts.FullConfigOutputPath) - return - } - opts.DryRun = *dryRun != "false" logrus.WithField("config", *configPath).WithField("version", version.Version).Printf("starting") @@ -659,128 +649,3 @@ func exportHistoricPrices(dayStart uint64, dayEnd uint64) { logrus.Info("historic price update run completed") } - -func generateConfigFromTestnetStub(configPath string, fullConfigOutputPath string) { - - data, err := os.ReadFile(configPath) - - if err != nil { - logrus.Fatalf("error reading config file: %v", err) - } - - stub := &types.LocalTestnetConfigStub{} - - err = yaml.Unmarshal(data, &stub) - if err != nil { - logrus.Fatalf("error parsing config stub: %v", err) - } - - if stub.PRESETBASE == "mainnet" { - cfg := &types.ChainConfig{} - - err := yaml.Unmarshal([]byte(config.MainnetChainYml), cfg) - - if err != nil { - logrus.Fatalf("error initializing full config: %v", err) - } - - cfg.TerminalTotalDifficulty = stub.TERMINALTOTALDIFFICULTY - cfg.TerminalBlockHash = stub.TERMINALBLOCKHASH - cfg.TerminalBlockHashActivationEpoch, err = strconv.ParseUint(stub.TERMINALBLOCKHASHACTIVATIONEPOCH, 10, 64) - if err != nil { - logrus.Fatalf("error overwriting ALTAIRFORKEPOCH: %v", err) - } - cfg.MinGenesisActiveValidatorCount, err = strconv.ParseUint(stub.MINGENESISACTIVEVALIDATORCOUNT, 10, 64) - if err != nil { - logrus.Fatalf("error overwriting ALTAIRFORKEPOCH: %v", err) - } - cfg.MinGenesisTime, err = strconv.ParseInt(stub.MINGENESISTIME, 10, 64) - if err != nil { - logrus.Fatalf("error overwriting ALTAIRFORKEPOCH: %v", err) - } - cfg.GenesisForkVersion = stub.GENESISFORKVERSION - cfg.GenesisDelay, err = strconv.ParseUint(stub.GENESISDELAY, 10, 64) - if err != nil { - logrus.Fatalf("error overwriting ALTAIRFORKEPOCH: %v", err) - } - cfg.AltairForkVersion = stub.ALTAIRFORKVERSION - cfg.AltairForkEpoch, err = strconv.ParseUint(stub.ALTAIRFORKEPOCH, 10, 64) - if err != nil { - logrus.Fatalf("error overwriting ALTAIRFORKEPOCH: %v", err) - } - cfg.BellatrixForkVersion = stub.BELLATRIXFORKEPOCH - cfg.BellatrixForkEpoch, err = strconv.ParseUint(stub.BELLATRIXFORKEPOCH, 10, 64) - if err != nil { - logrus.Fatalf("error overwriting ALTAIRFORKEPOCH: %v", err) - } - cfg.CappellaForkVersion = stub.CAPELLAFORKVERSION - cfg.CappellaForkEpoch, err = strconv.ParseUint(stub.CAPELLAFORKEPOCH, 10, 64) - if err != nil { - logrus.Fatalf("error overwriting ALTAIRFORKEPOCH: %v", err) - } - cfg.SecondsPerSlot, err = strconv.ParseUint(stub.SECONDSPERSLOT, 10, 64) - if err != nil { - logrus.Fatalf("error overwriting ALTAIRFORKEPOCH: %v", err) - } - cfg.SecondsPerEth1Block, err = strconv.ParseUint(stub.SECONDSPERETH1BLOCK, 10, 64) - if err != nil { - logrus.Fatalf("error overwriting ALTAIRFORKEPOCH: %v", err) - } - cfg.MinValidatorWithdrawabilityDelay, err = strconv.ParseUint(stub.MINVALIDATORWITHDRAWABILITYDELAY, 10, 64) - if err != nil { - logrus.Fatalf("error overwriting ALTAIRFORKEPOCH: %v", err) - } - cfg.ShardCommitteePeriod, err = strconv.ParseUint(stub.SHARDCOMMITTEEPERIOD, 10, 64) - if err != nil { - logrus.Fatalf("error overwriting ALTAIRFORKEPOCH: %v", err) - } - cfg.Eth1FollowDistance, err = strconv.ParseUint(stub.ETH1FOLLOWDISTANCE, 10, 64) - if err != nil { - logrus.Fatalf("error overwriting ALTAIRFORKEPOCH: %v", err) - } - cfg.InactivityScoreBias, err = strconv.ParseUint(stub.INACTIVITYSCOREBIAS, 10, 64) - if err != nil { - logrus.Fatalf("error overwriting ALTAIRFORKEPOCH: %v", err) - } - cfg.InactivityScoreRecoveryRate, err = strconv.ParseUint(stub.INACTIVITYSCORERECOVERYRATE, 10, 64) - if err != nil { - logrus.Fatalf("error overwriting ALTAIRFORKEPOCH: %v", err) - } - cfg.EjectionBalance, err = strconv.ParseUint(stub.EJECTIONBALANCE, 10, 64) - if err != nil { - logrus.Fatalf("error overwriting ALTAIRFORKEPOCH: %v", err) - } - cfg.MinPerEpochChurnLimit, err = strconv.ParseUint(stub.MINPEREPOCHCHURNLIMIT, 10, 64) - if err != nil { - logrus.Fatalf("error overwriting ALTAIRFORKEPOCH: %v", err) - } - cfg.ChurnLimitQuotient, err = strconv.ParseUint(stub.CHURNLIMITQUOTIENT, 10, 64) - if err != nil { - logrus.Fatalf("error overwriting ALTAIRFORKEPOCH: %v", err) - } - cfg.ProposerScoreBoost, err = strconv.ParseUint(stub.PROPOSERSCOREBOOST, 10, 64) - if err != nil { - logrus.Fatalf("error overwriting ALTAIRFORKEPOCH: %v", err) - } - cfg.DepositChainID, err = strconv.ParseUint(stub.DEPOSITCHAINID, 10, 64) - if err != nil { - logrus.Fatalf("error overwriting ALTAIRFORKEPOCH: %v", err) - } - cfg.DepositNetworkID, err = strconv.ParseUint(stub.DEPOSITNETWORKID, 10, 64) - if err != nil { - logrus.Fatalf("error overwriting ALTAIRFORKEPOCH: %v", err) - } - cfg.DepositContractAddress = stub.DEPOSITCONTRACTADDRESS - - encodedConfig, err := yaml.Marshal(cfg) - if err != nil { - logrus.Fatalf("error encoding new chain config: %v", err) - } - - err = os.WriteFile(fullConfigOutputPath, encodedConfig, 0664) - if err != nil { - logrus.Fatalf("error writing new chain config: %v", err) - } - } - -} diff --git a/go.mod b/go.mod index 5bc799320f..6cf7a94c09 100644 --- a/go.mod +++ b/go.mod @@ -80,6 +80,7 @@ require ( cloud.google.com/go/iam v0.7.0 // indirect cloud.google.com/go/longrunning v0.3.0 // indirect github.com/alessio/shellescape v1.4.1 // indirect + github.com/carlmjohnson/requests v0.23.4 // indirect github.com/census-instrumentation/opencensus-proto v0.2.1 // indirect github.com/cncf/udpa/go v0.0.0-20210930031921-04548b0d99d4 // indirect github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1 // indirect diff --git a/go.sum b/go.sum index 730fe57261..59ed5a5af9 100644 --- a/go.sum +++ b/go.sum @@ -106,6 +106,8 @@ github.com/btcsuite/btcd/btcec/v2 v2.3.2 h1:5n0X6hX0Zk+6omWcihdYvdAlGf2DfasC0GMf github.com/btcsuite/btcd/btcec/v2 v2.3.2/go.mod h1:zYzJ8etWJQIv1Ogk7OzpWjowwOdXY1W/17j2MW85J04= github.com/btcsuite/btcd/chaincfg/chainhash v1.0.1 h1:q0rUy8C/TYNBQS1+CGKw68tLOFYSNEs0TFnxxnS9+4U= github.com/bufbuild/buf v0.37.0/go.mod h1:lQ1m2HkIaGOFba6w/aC3KYBHhKEOESP3gaAEpS3dAFM= +github.com/carlmjohnson/requests v0.23.4 h1:AxcvapfB9RPXLSyvAHk9YJoodQ43ZjzNHj6Ft3tQGdg= +github.com/carlmjohnson/requests v0.23.4/go.mod h1:Qzp6tW4DQyainPP+tGwiJTzwxvElTIKm0B191TgTtOA= github.com/census-instrumentation/opencensus-proto v0.2.1 h1:glEXhBS5PSLLv4IXzLA5yPRVX4bilULVyxxbrfOtDAk= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/cp v1.1.1 h1:nCb6ZLdB7NRaqsm91JtQTAme2SKJzXVsdPIPkyJr1MU= diff --git a/local-deployment/README.md b/local-deployment/README.md index e4bcd69143..4c2711efc0 100644 --- a/local-deployment/README.md +++ b/local-deployment/README.md @@ -5,16 +5,22 @@ This guide outlines how to deploy the explorer using a local lh-geth testnet. Ut sudo apt update sudo apt-get install ca-certificates curl gnupg sudo install -m 0755 -d /etc/apt/keyrings -curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo gpg --dearmor -o /etc/apt/keyrings/docker.gpg +curl -fsSL https://download.docker.com/linux/debian/gpg | sudo gpg --dearmor -o /etc/apt/keyrings/docker.gpg sudo chmod a+r /etc/apt/keyrings/docker.gpg echo \ - "deb [arch="$(dpkg --print-architecture)" signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/ubuntu \ + "deb [arch="$(dpkg --print-architecture)" signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/debian \ "$(. /etc/os-release && echo "$VERSION_CODENAME")" stable" | \ sudo tee /etc/apt/sources.list.d/docker.list > /dev/null sudo apt-get update sudo apt-get install docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin sudo usermod -aG docker $USER ``` +# Install kurtosis-cli +``` +echo "deb [trusted=yes] https://apt.fury.io/kurtosis-tech/ /" | sudo tee /etc/apt/sources.list.d/kurtosis.list +sudo apt update +sudo apt install kurtosis-cli +``` # Install golang ``` wget https://go.dev/dl/go1.20.7.linux-amd64.tar.gz @@ -35,36 +41,6 @@ cd testnet sudo apt remove google-cloud-cli sudo apt install google-cloud-sdk-cbt ``` -# Clone the lh repo -``` -git clone https://github.com/sigp/lighthouse.git -cd lighthouse -``` -``` -# setup rust dev environment -curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -source "$HOME/.cargo/env" -``` -# Install lh build deps -``` -sudo apt install -y git gcc g++ make cmake pkg-config llvm-dev libclang-dev clang protobuf-compiler jq -``` -# Build & install lighthouse -``` -make -``` -# Build & install lcli -``` -make install-lcli -``` -# Download and install geth & bootnode binary -``` -cd ~ -wget https://gethstore.blob.core.windows.net/builds/geth-alltools-linux-amd64-1.12.2-bed84606.tar.gz -tar -xf geth-alltools-linux-amd64-1.12.2-bed84606.tar.gz -sudo cp geth-alltools-linux-amd64-1.12.2-bed84606/geth /usr/bin/ -sudo cp geth-alltools-linux-amd64-1.12.2-bed84606/bootnode /usr/bin/ -``` # Clone the explorer repository ``` cd ~/ @@ -103,18 +79,8 @@ lbt will be available on http://127.0.0.1:9000 bash ~/eth2-beaconchain-explorer/local-deployment/init-bigtable.sh ``` # Start up the local testnet nodes -## Switch to the lighthous scripts directory -``` -cd testnet/lighthouse/scripts/local_testnet/ -``` -## Start the local testnet -``` -cd testnet/lighthouse/scripts/local_testnet/ -./start_local_testnet.sh -v 2 genesis.json -``` -## Generate the explorer config from the lh testnet config stub ``` -eth2-beaconchain-explorer/bin/misc -command generate-config-from-testnet-stub -config .lighthouse/local-testnet/testnet/config.yaml -output-path .lighthouse/local-testnet/testnet/config.full.yaml +kurtosis run --enclave my-testnet github.com/kurtosis-tech/eth-network-package ``` # Initialize the db schema ``` @@ -122,13 +88,13 @@ BIGTABLE_EMULATOR_HOST="127.0.0.1:9000" ~/eth2-beaconchain-explorer/bin/misc -co ``` # Start the indexer ``` -CHAIN_GENESIS_TIMESTAMP= BIGTABLE_EMULATOR_HOST="127.0.0.1:9000" ~/eth2-beaconchain-explorer/bin/explorer -config ~/eth2-beaconchain-explorer/local-deployment/testnet-config.yml +BIGTABLE_EMULATOR_HOST="127.0.0.1:9000" ~/eth2-beaconchain-explorer/bin/explorer -config ~/eth2-beaconchain-explorer/local-deployment/testnet-config.yml ``` -# Start the fdu +# Start the frontend-data-updater ``` -CHAIN_GENESIS_TIMESTAMP= BIGTABLE_EMULATOR_HOST="127.0.0.1:9000" ~/eth2-beaconchain-explorer/bin/frontend-data-updater -config ~/eth2-beaconchain-explorer/local-deployment/testnet-config.yml +BIGTABLE_EMULATOR_HOST="127.0.0.1:9000" ~/eth2-beaconchain-explorer/bin/frontend-data-updater -config ~/eth2-beaconchain-explorer/local-deployment/testnet-config.yml ``` # Start the frontend ``` -CHAIN_GENESIS_TIMESTAMP= BIGTABLE_EMULATOR_HOST="127.0.0.1:9000" ~/eth2-beaconchain-explorer/bin/frontend-data-updater -config ~/eth2-beaconchain-explorer/local-deployment/testnet-config-frontend.yml +BIGTABLE_EMULATOR_HOST="127.0.0.1:9000" ~/eth2-beaconchain-explorer/bin/frontend-data-updater -config ~/eth2-beaconchain-explorer/local-deployment/testnet-config-frontend.yml ``` diff --git a/local-deployment/testnet-config-frontend.yml b/local-deployment/testnet-config-frontend.yml index f3d660f08c..322db49509 100644 --- a/local-deployment/testnet-config-frontend.yml +++ b/local-deployment/testnet-config-frontend.yml @@ -1,7 +1,5 @@ chain: name: local-testnet - configPath: '.lighthouse/local-testnet/testnet/config.full.yaml' - genesisTimestamp: 1693397704 readerDatabase: name: db host: 127.0.0.1 diff --git a/local-deployment/testnet-config.yml b/local-deployment/testnet-config.yml index 5326152bf3..8d94a99e6d 100644 --- a/local-deployment/testnet-config.yml +++ b/local-deployment/testnet-config.yml @@ -1,7 +1,5 @@ chain: - name: local-testnet - configPath: '.lighthouse/local-testnet/testnet/config.full.yaml' - genesisTimestamp: 1693397704 + configPath: 'node' readerDatabase: name: db host: 127.0.0.1 diff --git a/types/config.go b/types/config.go index a44d57bfe0..1aede97a0c 100644 --- a/types/config.go +++ b/types/config.go @@ -217,36 +217,102 @@ type ServiceMonitoringConfiguration struct { Duration time.Duration `yaml:"duration" envconfig:"DURATION"` } -type LocalTestnetConfigStub struct { - CONFIGNAME string `yaml:"CONFIG_NAME"` - PRESETBASE string `yaml:"PRESET_BASE"` - TERMINALTOTALDIFFICULTY string `yaml:"TERMINAL_TOTAL_DIFFICULTY"` - TERMINALBLOCKHASH string `yaml:"TERMINAL_BLOCK_HASH"` - TERMINALBLOCKHASHACTIVATIONEPOCH string `yaml:"TERMINAL_BLOCK_HASH_ACTIVATION_EPOCH"` - SAFESLOTSTOIMPORTOPTIMISTICALLY string `yaml:"SAFE_SLOTS_TO_IMPORT_OPTIMISTICALLY"` - MINGENESISACTIVEVALIDATORCOUNT string `yaml:"MIN_GENESIS_ACTIVE_VALIDATOR_COUNT"` - MINGENESISTIME string `yaml:"MIN_GENESIS_TIME"` - GENESISFORKVERSION string `yaml:"GENESIS_FORK_VERSION"` - GENESISDELAY string `yaml:"GENESIS_DELAY"` - ALTAIRFORKVERSION string `yaml:"ALTAIR_FORK_VERSION"` - ALTAIRFORKEPOCH string `yaml:"ALTAIR_FORK_EPOCH"` - BELLATRIXFORKVERSION string `yaml:"BELLATRIX_FORK_VERSION"` - BELLATRIXFORKEPOCH string `yaml:"BELLATRIX_FORK_EPOCH"` - CAPELLAFORKVERSION string `yaml:"CAPELLA_FORK_VERSION"` - CAPELLAFORKEPOCH string `yaml:"CAPELLA_FORK_EPOCH"` - SECONDSPERSLOT string `yaml:"SECONDS_PER_SLOT"` - SECONDSPERETH1BLOCK string `yaml:"SECONDS_PER_ETH1_BLOCK"` - MINVALIDATORWITHDRAWABILITYDELAY string `yaml:"MIN_VALIDATOR_WITHDRAWABILITY_DELAY"` - SHARDCOMMITTEEPERIOD string `yaml:"SHARD_COMMITTEE_PERIOD"` - ETH1FOLLOWDISTANCE string `yaml:"ETH1_FOLLOW_DISTANCE"` - SUBNETSPERNODE string `yaml:"SUBNETS_PER_NODE"` - INACTIVITYSCOREBIAS string `yaml:"INACTIVITY_SCORE_BIAS"` - INACTIVITYSCORERECOVERYRATE string `yaml:"INACTIVITY_SCORE_RECOVERY_RATE"` - EJECTIONBALANCE string `yaml:"EJECTION_BALANCE"` - MINPEREPOCHCHURNLIMIT string `yaml:"MIN_PER_EPOCH_CHURN_LIMIT"` - CHURNLIMITQUOTIENT string `yaml:"CHURN_LIMIT_QUOTIENT"` - PROPOSERSCOREBOOST string `yaml:"PROPOSER_SCORE_BOOST"` - DEPOSITCHAINID string `yaml:"DEPOSIT_CHAIN_ID"` - DEPOSITNETWORKID string `yaml:"DEPOSIT_NETWORK_ID"` - DEPOSITCONTRACTADDRESS string `yaml:"DEPOSIT_CONTRACT_ADDRESS"` +type ConfigJsonResponse struct { + Data struct { + ConfigName string `json:"CONFIG_NAME"` + PresetBase string `json:"PRESET_BASE"` + TerminalTotalDifficulty string `json:"TERMINAL_TOTAL_DIFFICULTY"` + TerminalBlockHash string `json:"TERMINAL_BLOCK_HASH"` + TerminalBlockHashActivationEpoch string `json:"TERMINAL_BLOCK_HASH_ACTIVATION_EPOCH"` + SafeSlotsToImportOptimistically string `json:"SAFE_SLOTS_TO_IMPORT_OPTIMISTICALLY"` + MinGenesisActiveValidatorCount string `json:"MIN_GENESIS_ACTIVE_VALIDATOR_COUNT"` + MinGenesisTime string `json:"MIN_GENESIS_TIME"` + GenesisForkVersion string `json:"GENESIS_FORK_VERSION"` + GenesisDelay string `json:"GENESIS_DELAY"` + AltairForkVersion string `json:"ALTAIR_FORK_VERSION"` + AltairForkEpoch string `json:"ALTAIR_FORK_EPOCH"` + BellatrixForkVersion string `json:"BELLATRIX_FORK_VERSION"` + BellatrixForkEpoch string `json:"BELLATRIX_FORK_EPOCH"` + CapellaForkVersion string `json:"CAPELLA_FORK_VERSION"` + CapellaForkEpoch string `json:"CAPELLA_FORK_EPOCH"` + SecondsPerSlot string `json:"SECONDS_PER_SLOT"` + SecondsPerEth1Block string `json:"SECONDS_PER_ETH1_BLOCK"` + MinValidatorWithdrawabilityDelay string `json:"MIN_VALIDATOR_WITHDRAWABILITY_DELAY"` + ShardCommitteePeriod string `json:"SHARD_COMMITTEE_PERIOD"` + Eth1FollowDistance string `json:"ETH1_FOLLOW_DISTANCE"` + SubnetsPerNode string `json:"SUBNETS_PER_NODE"` + InactivityScoreBias string `json:"INACTIVITY_SCORE_BIAS"` + InactivityScoreRecoveryRate string `json:"INACTIVITY_SCORE_RECOVERY_RATE"` + EjectionBalance string `json:"EJECTION_BALANCE"` + MinPerEpochChurnLimit string `json:"MIN_PER_EPOCH_CHURN_LIMIT"` + ChurnLimitQuotient string `json:"CHURN_LIMIT_QUOTIENT"` + ProposerScoreBoost string `json:"PROPOSER_SCORE_BOOST"` + DepositChainID string `json:"DEPOSIT_CHAIN_ID"` + DepositNetworkID string `json:"DEPOSIT_NETWORK_ID"` + DepositContractAddress string `json:"DEPOSIT_CONTRACT_ADDRESS"` + MaxCommitteesPerSlot string `json:"MAX_COMMITTEES_PER_SLOT"` + TargetCommitteeSize string `json:"TARGET_COMMITTEE_SIZE"` + MaxValidatorsPerCommittee string `json:"MAX_VALIDATORS_PER_COMMITTEE"` + ShuffleRoundCount string `json:"SHUFFLE_ROUND_COUNT"` + HysteresisQuotient string `json:"HYSTERESIS_QUOTIENT"` + HysteresisDownwardMultiplier string `json:"HYSTERESIS_DOWNWARD_MULTIPLIER"` + HysteresisUpwardMultiplier string `json:"HYSTERESIS_UPWARD_MULTIPLIER"` + SafeSlotsToUpdateJustified string `json:"SAFE_SLOTS_TO_UPDATE_JUSTIFIED"` + MinDepositAmount string `json:"MIN_DEPOSIT_AMOUNT"` + MaxEffectiveBalance string `json:"MAX_EFFECTIVE_BALANCE"` + EffectiveBalanceIncrement string `json:"EFFECTIVE_BALANCE_INCREMENT"` + MinAttestationInclusionDelay string `json:"MIN_ATTESTATION_INCLUSION_DELAY"` + SlotsPerEpoch string `json:"SLOTS_PER_EPOCH"` + MinSeedLookahead string `json:"MIN_SEED_LOOKAHEAD"` + MaxSeedLookahead string `json:"MAX_SEED_LOOKAHEAD"` + EpochsPerEth1VotingPeriod string `json:"EPOCHS_PER_ETH1_VOTING_PERIOD"` + SlotsPerHistoricalRoot string `json:"SLOTS_PER_HISTORICAL_ROOT"` + MinEpochsToInactivityPenalty string `json:"MIN_EPOCHS_TO_INACTIVITY_PENALTY"` + EpochsPerHistoricalVector string `json:"EPOCHS_PER_HISTORICAL_VECTOR"` + EpochsPerSlashingsVector string `json:"EPOCHS_PER_SLASHINGS_VECTOR"` + HistoricalRootsLimit string `json:"HISTORICAL_ROOTS_LIMIT"` + ValidatorRegistryLimit string `json:"VALIDATOR_REGISTRY_LIMIT"` + BaseRewardFactor string `json:"BASE_REWARD_FACTOR"` + WhistleblowerRewardQuotient string `json:"WHISTLEBLOWER_REWARD_QUOTIENT"` + ProposerRewardQuotient string `json:"PROPOSER_REWARD_QUOTIENT"` + InactivityPenaltyQuotient string `json:"INACTIVITY_PENALTY_QUOTIENT"` + MinSlashingPenaltyQuotient string `json:"MIN_SLASHING_PENALTY_QUOTIENT"` + ProportionalSlashingMultiplier string `json:"PROPORTIONAL_SLASHING_MULTIPLIER"` + MaxProposerSlashings string `json:"MAX_PROPOSER_SLASHINGS"` + MaxAttesterSlashings string `json:"MAX_ATTESTER_SLASHINGS"` + MaxAttestations string `json:"MAX_ATTESTATIONS"` + MaxDeposits string `json:"MAX_DEPOSITS"` + MaxVoluntaryExits string `json:"MAX_VOLUNTARY_EXITS"` + InactivityPenaltyQuotientAltair string `json:"INACTIVITY_PENALTY_QUOTIENT_ALTAIR"` + MinSlashingPenaltyQuotientAltair string `json:"MIN_SLASHING_PENALTY_QUOTIENT_ALTAIR"` + ProportionalSlashingMultiplierAltair string `json:"PROPORTIONAL_SLASHING_MULTIPLIER_ALTAIR"` + SyncCommitteeSize string `json:"SYNC_COMMITTEE_SIZE"` + EpochsPerSyncCommitteePeriod string `json:"EPOCHS_PER_SYNC_COMMITTEE_PERIOD"` + MinSyncCommitteeParticipants string `json:"MIN_SYNC_COMMITTEE_PARTICIPANTS"` + InactivityPenaltyQuotientBellatrix string `json:"INACTIVITY_PENALTY_QUOTIENT_BELLATRIX"` + MinSlashingPenaltyQuotientBellatrix string `json:"MIN_SLASHING_PENALTY_QUOTIENT_BELLATRIX"` + ProportionalSlashingMultiplierBellatrix string `json:"PROPORTIONAL_SLASHING_MULTIPLIER_BELLATRIX"` + MaxBytesPerTransaction string `json:"MAX_BYTES_PER_TRANSACTION"` + MaxTransactionsPerPayload string `json:"MAX_TRANSACTIONS_PER_PAYLOAD"` + BytesPerLogsBloom string `json:"BYTES_PER_LOGS_BLOOM"` + MaxExtraDataBytes string `json:"MAX_EXTRA_DATA_BYTES"` + MaxBlsToExecutionChanges string `json:"MAX_BLS_TO_EXECUTION_CHANGES"` + MaxWithdrawalsPerPayload string `json:"MAX_WITHDRAWALS_PER_PAYLOAD"` + MaxValidatorsPerWithdrawalsSweep string `json:"MAX_VALIDATORS_PER_WITHDRAWALS_SWEEP"` + DomainAggregateAndProof string `json:"DOMAIN_AGGREGATE_AND_PROOF"` + TargetAggregatorsPerSyncSubcommittee string `json:"TARGET_AGGREGATORS_PER_SYNC_SUBCOMMITTEE"` + SyncCommitteeSubnetCount string `json:"SYNC_COMMITTEE_SUBNET_COUNT"` + BlsWithdrawalPrefix string `json:"BLS_WITHDRAWAL_PREFIX"` + DomainRandao string `json:"DOMAIN_RANDAO"` + DomainVoluntaryExit string `json:"DOMAIN_VOLUNTARY_EXIT"` + DomainSyncCommitteeSelectionProof string `json:"DOMAIN_SYNC_COMMITTEE_SELECTION_PROOF"` + DomainBeaconAttester string `json:"DOMAIN_BEACON_ATTESTER"` + DomainBeaconProposer string `json:"DOMAIN_BEACON_PROPOSER"` + DomainDeposit string `json:"DOMAIN_DEPOSIT"` + DomainSelectionProof string `json:"DOMAIN_SELECTION_PROOF"` + DomainSyncCommittee string `json:"DOMAIN_SYNC_COMMITTEE"` + TargetAggregatorsPerCommittee string `json:"TARGET_AGGREGATORS_PER_COMMITTEE"` + DomainContributionAndProof string `json:"DOMAIN_CONTRIBUTION_AND_PROOF"` + DomainApplicationMask string `json:"DOMAIN_APPLICATION_MASK"` + } `json:"data"` } diff --git a/utils/utils.go b/utils/utils.go index 028c212d52..154bef010a 100644 --- a/utils/utils.go +++ b/utils/utils.go @@ -3,6 +3,7 @@ package utils import ( "bufio" "bytes" + "context" securerand "crypto/rand" "crypto/sha256" "database/sql" @@ -28,6 +29,7 @@ import ( "regexp" "runtime" "sort" + "strconv" "strings" "time" "unicode/utf8" @@ -38,6 +40,7 @@ import ( "gopkg.in/yaml.v3" "github.com/asaskevich/govalidator" + "github.com/carlmjohnson/requests" "github.com/ethereum/go-ethereum/accounts/abi" "github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethereum/go-ethereum/params" @@ -419,6 +422,126 @@ func ReadConfig(cfg *types.Config, path string) error { // if err != nil { // return fmt.Errorf("error setting chainConfig (%v) for prysmParams: %w", cfg.Chain.Name, err) // } + } else if cfg.Chain.ConfigPath == "node" { + nodeEndpoint := fmt.Sprintf("http://%s:%s", cfg.Indexer.Node.Host, cfg.Indexer.Node.Port) + + jr := &types.ConfigJsonResponse{} + + err := requests. + URL(nodeEndpoint + "/eth/v1/config/spec"). + ToJSON(jr). + Fetch(context.Background()) + + if err != nil { + return err + } + + chainCfg := types.ChainConfig{ + PresetBase: jr.Data.PresetBase, + ConfigName: jr.Data.ConfigName, + TerminalTotalDifficulty: jr.Data.TerminalTotalDifficulty, + TerminalBlockHash: jr.Data.TerminalBlockHash, + TerminalBlockHashActivationEpoch: mustParseUint(jr.Data.TerminalBlockHashActivationEpoch), + MinGenesisActiveValidatorCount: mustParseUint(jr.Data.MinGenesisActiveValidatorCount), + MinGenesisTime: int64(mustParseUint(jr.Data.MinGenesisTime)), + GenesisForkVersion: jr.Data.GenesisForkVersion, + GenesisDelay: mustParseUint(jr.Data.GenesisDelay), + AltairForkVersion: jr.Data.AltairForkVersion, + AltairForkEpoch: mustParseUint(jr.Data.AltairForkEpoch), + BellatrixForkVersion: jr.Data.BellatrixForkVersion, + BellatrixForkEpoch: mustParseUint(jr.Data.BellatrixForkEpoch), + CappellaForkVersion: jr.Data.CapellaForkVersion, + CappellaForkEpoch: mustParseUint(jr.Data.CapellaForkEpoch), + SecondsPerSlot: mustParseUint(jr.Data.SecondsPerSlot), + SecondsPerEth1Block: mustParseUint(jr.Data.SecondsPerEth1Block), + MinValidatorWithdrawabilityDelay: mustParseUint(jr.Data.MinValidatorWithdrawabilityDelay), + ShardCommitteePeriod: mustParseUint(jr.Data.ShardCommitteePeriod), + Eth1FollowDistance: mustParseUint(jr.Data.Eth1FollowDistance), + InactivityScoreBias: mustParseUint(jr.Data.InactivityScoreBias), + InactivityScoreRecoveryRate: mustParseUint(jr.Data.InactivityScoreRecoveryRate), + EjectionBalance: mustParseUint(jr.Data.EjectionBalance), + MinPerEpochChurnLimit: mustParseUint(jr.Data.MinPerEpochChurnLimit), + ChurnLimitQuotient: mustParseUint(jr.Data.ChurnLimitQuotient), + ProposerScoreBoost: mustParseUint(jr.Data.ProposerScoreBoost), + DepositChainID: mustParseUint(jr.Data.DepositChainID), + DepositNetworkID: mustParseUint(jr.Data.DepositNetworkID), + DepositContractAddress: jr.Data.DepositContractAddress, + MaxCommitteesPerSlot: mustParseUint(jr.Data.MaxCommitteesPerSlot), + TargetCommitteeSize: mustParseUint(jr.Data.TargetCommitteeSize), + MaxValidatorsPerCommittee: mustParseUint(jr.Data.TargetCommitteeSize), + ShuffleRoundCount: mustParseUint(jr.Data.ShuffleRoundCount), + HysteresisQuotient: mustParseUint(jr.Data.HysteresisQuotient), + HysteresisDownwardMultiplier: mustParseUint(jr.Data.HysteresisDownwardMultiplier), + HysteresisUpwardMultiplier: mustParseUint(jr.Data.HysteresisUpwardMultiplier), + SafeSlotsToUpdateJustified: mustParseUint(jr.Data.SafeSlotsToUpdateJustified), + MinDepositAmount: mustParseUint(jr.Data.MinDepositAmount), + MaxEffectiveBalance: mustParseUint(jr.Data.MaxEffectiveBalance), + EffectiveBalanceIncrement: mustParseUint(jr.Data.EffectiveBalanceIncrement), + MinAttestationInclusionDelay: mustParseUint(jr.Data.MinAttestationInclusionDelay), + SlotsPerEpoch: mustParseUint(jr.Data.SlotsPerEpoch), + MinSeedLookahead: mustParseUint(jr.Data.MinSeedLookahead), + MaxSeedLookahead: mustParseUint(jr.Data.MaxSeedLookahead), + EpochsPerEth1VotingPeriod: mustParseUint(jr.Data.EpochsPerEth1VotingPeriod), + SlotsPerHistoricalRoot: mustParseUint(jr.Data.SlotsPerHistoricalRoot), + MinEpochsToInactivityPenalty: mustParseUint(jr.Data.MinEpochsToInactivityPenalty), + EpochsPerHistoricalVector: mustParseUint(jr.Data.EpochsPerHistoricalVector), + EpochsPerSlashingsVector: mustParseUint(jr.Data.EpochsPerSlashingsVector), + HistoricalRootsLimit: mustParseUint(jr.Data.HistoricalRootsLimit), + ValidatorRegistryLimit: mustParseUint(jr.Data.ValidatorRegistryLimit), + BaseRewardFactor: mustParseUint(jr.Data.BaseRewardFactor), + WhistleblowerRewardQuotient: mustParseUint(jr.Data.WhistleblowerRewardQuotient), + ProposerRewardQuotient: mustParseUint(jr.Data.ProposerRewardQuotient), + InactivityPenaltyQuotient: mustParseUint(jr.Data.InactivityPenaltyQuotient), + MinSlashingPenaltyQuotient: mustParseUint(jr.Data.MinSlashingPenaltyQuotient), + ProportionalSlashingMultiplier: mustParseUint(jr.Data.ProportionalSlashingMultiplier), + MaxProposerSlashings: mustParseUint(jr.Data.MaxProposerSlashings), + MaxAttesterSlashings: mustParseUint(jr.Data.MaxAttesterSlashings), + MaxAttestations: mustParseUint(jr.Data.MaxAttestations), + MaxDeposits: mustParseUint(jr.Data.MaxDeposits), + MaxVoluntaryExits: mustParseUint(jr.Data.MaxVoluntaryExits), + InvactivityPenaltyQuotientAltair: mustParseUint(jr.Data.InactivityPenaltyQuotientAltair), + MinSlashingPenaltyQuotientAltair: mustParseUint(jr.Data.MinSlashingPenaltyQuotientAltair), + ProportionalSlashingMultiplierAltair: mustParseUint(jr.Data.ProportionalSlashingMultiplierAltair), + SyncCommitteeSize: mustParseUint(jr.Data.SyncCommitteeSize), + EpochsPerSyncCommitteePeriod: mustParseUint(jr.Data.EpochsPerSyncCommitteePeriod), + MinSyncCommitteeParticipants: mustParseUint(jr.Data.MinSyncCommitteeParticipants), + InvactivityPenaltyQuotientBellatrix: mustParseUint(jr.Data.InactivityPenaltyQuotientBellatrix), + MinSlashingPenaltyQuotientBellatrix: mustParseUint(jr.Data.MinSlashingPenaltyQuotientBellatrix), + ProportionalSlashingMultiplierBellatrix: mustParseUint(jr.Data.ProportionalSlashingMultiplierBellatrix), + MaxBytesPerTransaction: mustParseUint(jr.Data.MaxBytesPerTransaction), + MaxTransactionsPerPayload: mustParseUint(jr.Data.MaxTransactionsPerPayload), + BytesPerLogsBloom: mustParseUint(jr.Data.BytesPerLogsBloom), + MaxExtraDataBytes: mustParseUint(jr.Data.MaxExtraDataBytes), + MaxWithdrawalsPerPayload: mustParseUint(jr.Data.MaxWithdrawalsPerPayload), + MaxValidatorsPerWithdrawalSweep: mustParseUint(jr.Data.MaxValidatorsPerWithdrawalsSweep), + MaxBlsToExecutionChange: mustParseUint(jr.Data.MaxBlsToExecutionChanges), + } + + cfg.Chain.Config = chainCfg + + type GenesisResponse struct { + Data struct { + GenesisTime string `json:"genesis_time"` + GenesisValidatorsRoot string `json:"genesis_validators_root"` + GenesisForkVersion string `json:"genesis_fork_version"` + } `json:"data"` + } + + gtr := &GenesisResponse{} + + err = requests. + URL(nodeEndpoint + "/eth/v1/beacon/genesis"). + ToJSON(jr). + Fetch(context.Background()) + + if err != nil { + return err + } + + cfg.Chain.GenesisTimestamp = mustParseUint(gtr.Data.GenesisTime) + + logger.Infof("loaded chain config from node") + } else { f, err := os.Open(cfg.Chain.ConfigPath) if err != nil { @@ -491,6 +614,15 @@ func ReadConfig(cfg *types.Config, path string) error { return nil } +func mustParseUint(str string) uint64 { + nbr, err := strconv.ParseUint(str, 10, 64) + if err != nil { + logrus.Fatalf("fatal error parsind uint %s: %v", str, err) + } + + return nbr +} + func readConfigFile(cfg *types.Config, path string) error { if path == "" { return yaml.Unmarshal([]byte(config.DefaultConfigYml), cfg) From 5087774b56ae628df6616acc180084dcfb0bce2e Mon Sep 17 00:00:00 2001 From: peter <1674920+peterbitfly@users.noreply.github.com> Date: Thu, 31 Aug 2023 09:17:29 +0200 Subject: [PATCH 027/106] (BIDS-2379) improve uint parsing --- local-deployment/README.md | 22 ++++------------------ local-deployment/docker-compose.yml | 7 +++++++ utils/utils.go | 7 ++++++- 3 files changed, 17 insertions(+), 19 deletions(-) diff --git a/local-deployment/README.md b/local-deployment/README.md index 4c2711efc0..fcb28c6068 100644 --- a/local-deployment/README.md +++ b/local-deployment/README.md @@ -49,31 +49,16 @@ cd eth2-beaconchain-explorer ``` # Build the explorer binaries ``` +sudo apt install build-essential make all make misc ``` -# Clone the little_bigtable repository -``` -cd ~/ -git clone https://github.com/gobitfly/little_bigtable.git -cd little_bigtable -``` -# Build the little_bigtable binary -``` -make -``` -# Start postgres & redis +# Start postgres, redis & little_bigtable ``` cd ~/eth2-beaconchain-explorer/local-deployment/ docker compose up -d ``` -Redis will be available on port 6379 and postgres on port 5432 (username `postgres`, password `pass`, db `db`) -# Start little_bigtable -``` -# in a new terminal -~/little_bigtable/build/little_bigtable -db-file ~/testnet/bigtable.db -``` -lbt will be available on http://127.0.0.1:9000 +Redis will be available on port 6379, little_bigtable on port 9000 and postgres on port 5432 (username `postgres`, password `pass`, db `db`) # Initialize the bigtable schema ``` bash ~/eth2-beaconchain-explorer/local-deployment/init-bigtable.sh @@ -82,6 +67,7 @@ bash ~/eth2-beaconchain-explorer/local-deployment/init-bigtable.sh ``` kurtosis run --enclave my-testnet github.com/kurtosis-tech/eth-network-package ``` +Take note of the http API ports of the el & cl clients. # Initialize the db schema ``` BIGTABLE_EMULATOR_HOST="127.0.0.1:9000" ~/eth2-beaconchain-explorer/bin/misc -config ~/eth2-beaconchain-explorer/local-deployment/testnet-config.yml -command applyDbSchema diff --git a/local-deployment/docker-compose.yml b/local-deployment/docker-compose.yml index 50cab77378..a62eec5a87 100644 --- a/local-deployment/docker-compose.yml +++ b/local-deployment/docker-compose.yml @@ -21,4 +21,11 @@ services: - ./tables.sql:/docker-entrypoint-initdb.d/init.sql ports: - 5432:5432 + little-bigtable: + image: gobitfly/little_bigtable + restart: unless-stopped + volumes: + - ./docker-volumes/little_bigtable:/app/data + ports: + - 9000:9000 diff --git a/utils/utils.go b/utils/utils.go index 154bef010a..d220ebfa51 100644 --- a/utils/utils.go +++ b/utils/utils.go @@ -615,9 +615,14 @@ func ReadConfig(cfg *types.Config, path string) error { } func mustParseUint(str string) uint64 { + + if str == "" { + return 0 + } + nbr, err := strconv.ParseUint(str, 10, 64) if err != nil { - logrus.Fatalf("fatal error parsind uint %s: %v", str, err) + logrus.Fatalf("fatal error parsing uint %s: %v", str, err) } return nbr From f9c90cc2103eb3369cb48f8dcace3fccf27749b5 Mon Sep 17 00:00:00 2001 From: peter <1674920+peterbitfly@users.noreply.github.com> Date: Thu, 31 Aug 2023 09:22:58 +0200 Subject: [PATCH 028/106] (BIDS-2379) improve logging --- go.mod | 2 +- local-deployment/README.md | 2 +- utils/utils.go | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 6cf7a94c09..268ba9369b 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module eth2-exporter -go 1.18 +go 1.20 require ( cloud.google.com/go/bigtable v1.16.0 diff --git a/local-deployment/README.md b/local-deployment/README.md index fcb28c6068..e8bb490e4b 100644 --- a/local-deployment/README.md +++ b/local-deployment/README.md @@ -70,7 +70,7 @@ kurtosis run --enclave my-testnet github.com/kurtosis-tech/eth-network-package Take note of the http API ports of the el & cl clients. # Initialize the db schema ``` -BIGTABLE_EMULATOR_HOST="127.0.0.1:9000" ~/eth2-beaconchain-explorer/bin/misc -config ~/eth2-beaconchain-explorer/local-deployment/testnet-config.yml -command applyDbSchema +INDEXER_NODE_PORT= BIGTABLE_EMULATOR_HOST="127.0.0.1:9000" ~/eth2-beaconchain-explorer/bin/misc -config ~/eth2-beaconchain-explorer/local-deployment/testnet-config.yml -command applyDbSchema ``` # Start the indexer ``` diff --git a/utils/utils.go b/utils/utils.go index d220ebfa51..4214970248 100644 --- a/utils/utils.go +++ b/utils/utils.go @@ -540,7 +540,7 @@ func ReadConfig(cfg *types.Config, path string) error { cfg.Chain.GenesisTimestamp = mustParseUint(gtr.Data.GenesisTime) - logger.Infof("loaded chain config from node") + logger.Infof("loaded chain config from node with genesis time %d", gtr.Data.GenesisTime) } else { f, err := os.Open(cfg.Chain.ConfigPath) From 8b2bf4ea1e943392c536f8d685a431cd50fe9915 Mon Sep 17 00:00:00 2001 From: peter <1674920+peterbitfly@users.noreply.github.com> Date: Thu, 31 Aug 2023 09:24:15 +0200 Subject: [PATCH 029/106] (BIDS-2379) fix genesis ts parsing --- utils/utils.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utils/utils.go b/utils/utils.go index 4214970248..fe93070e43 100644 --- a/utils/utils.go +++ b/utils/utils.go @@ -531,7 +531,7 @@ func ReadConfig(cfg *types.Config, path string) error { err = requests. URL(nodeEndpoint + "/eth/v1/beacon/genesis"). - ToJSON(jr). + ToJSON(gtr). Fetch(context.Background()) if err != nil { From d029fdeb4ef45c2e9a6196bb6ac75e96c8e1ac02 Mon Sep 17 00:00:00 2001 From: peter <1674920+peterbitfly@users.noreply.github.com> Date: Thu, 31 Aug 2023 09:24:25 +0200 Subject: [PATCH 030/106] (BIDS-2379) fix genesis ts formatting --- utils/utils.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utils/utils.go b/utils/utils.go index fe93070e43..8a77f78937 100644 --- a/utils/utils.go +++ b/utils/utils.go @@ -540,7 +540,7 @@ func ReadConfig(cfg *types.Config, path string) error { cfg.Chain.GenesisTimestamp = mustParseUint(gtr.Data.GenesisTime) - logger.Infof("loaded chain config from node with genesis time %d", gtr.Data.GenesisTime) + logger.Infof("loaded chain config from node with genesis time %s", gtr.Data.GenesisTime) } else { f, err := os.Open(cfg.Chain.ConfigPath) From b536000c08eddc62d7dc6e646ae3287d0d516b6d Mon Sep 17 00:00:00 2001 From: peter <1674920+peterbitfly@users.noreply.github.com> Date: Thu, 31 Aug 2023 09:26:16 +0200 Subject: [PATCH 031/106] (BIDS-2379) add genesis validator root provided by node --- utils/utils.go | 1 + 1 file changed, 1 insertion(+) diff --git a/utils/utils.go b/utils/utils.go index 8a77f78937..e79e89e3bd 100644 --- a/utils/utils.go +++ b/utils/utils.go @@ -539,6 +539,7 @@ func ReadConfig(cfg *types.Config, path string) error { } cfg.Chain.GenesisTimestamp = mustParseUint(gtr.Data.GenesisTime) + cfg.Chain.GenesisValidatorsRoot = gtr.Data.GenesisValidatorsRoot logger.Infof("loaded chain config from node with genesis time %s", gtr.Data.GenesisTime) From 3df4277bf61926afbe57f868d91a983299a05b7a Mon Sep 17 00:00:00 2001 From: peter <1674920+peterbitfly@users.noreply.github.com> Date: Thu, 31 Aug 2023 09:31:12 +0200 Subject: [PATCH 032/106] (BIDS-2379) improve config handling --- local-deployment/README.md | 4 ++ ...-config.yml => testnet-config-example.yml} | 8 +-- local-deployment/testnet-config-frontend.yml | 64 ------------------- 3 files changed, 7 insertions(+), 69 deletions(-) rename local-deployment/{testnet-config.yml => testnet-config-example.yml} (87%) delete mode 100644 local-deployment/testnet-config-frontend.yml diff --git a/local-deployment/README.md b/local-deployment/README.md index e8bb490e4b..82c69a7dad 100644 --- a/local-deployment/README.md +++ b/local-deployment/README.md @@ -68,6 +68,10 @@ bash ~/eth2-beaconchain-explorer/local-deployment/init-bigtable.sh kurtosis run --enclave my-testnet github.com/kurtosis-tech/eth-network-package ``` Take note of the http API ports of the el & cl clients. +# Copy the example config file and add the el & cl client ports +``` +cp ~/eth2-beaconchain-explorer/local-deployment/testnet-config-example.yml ~/testnet/explorer-config.yml +``` # Initialize the db schema ``` INDEXER_NODE_PORT= BIGTABLE_EMULATOR_HOST="127.0.0.1:9000" ~/eth2-beaconchain-explorer/bin/misc -config ~/eth2-beaconchain-explorer/local-deployment/testnet-config.yml -command applyDbSchema diff --git a/local-deployment/testnet-config.yml b/local-deployment/testnet-config-example.yml similarity index 87% rename from local-deployment/testnet-config.yml rename to local-deployment/testnet-config-example.yml index 8d94a99e6d..78089ab2f0 100644 --- a/local-deployment/testnet-config.yml +++ b/local-deployment/testnet-config-example.yml @@ -15,8 +15,8 @@ writerDatabase: bigtable: project: explorer instance: explorer -eth1ErigonEndpoint: 'http://127.0.0.1:6001' -eth1GethEndpoint: 'http://127.0.0.1:6001' +eth1ErigonEndpoint: 'http://127.0.0.1:' +eth1GethEndpoint: 'http://127.0.0.1:' redisCacheEndpoint: '127.0.0.1:6379' tieredCacheProvider: 'redis' #tieredCacheProvider: 'bigtable' @@ -56,8 +56,6 @@ indexer: # indexMissingEpochsOnStartup: true # Check for missing epochs and export them after startup node: host: 127.0.0.1 - port: '8001' + port: '' type: lighthouse - eth1Endpoint: "http://127.0.0.1:6001" - eth1DepositContractAddress: '0x4242424242424242424242424242424242424242' eth1DepositContractFirstBlock: 0 \ No newline at end of file diff --git a/local-deployment/testnet-config-frontend.yml b/local-deployment/testnet-config-frontend.yml deleted file mode 100644 index 322db49509..0000000000 --- a/local-deployment/testnet-config-frontend.yml +++ /dev/null @@ -1,64 +0,0 @@ -chain: - name: local-testnet -readerDatabase: - name: db - host: 127.0.0.1 - port: "5432" - user: postgres - password: "pass" -writerDatabase: - name: db - host: 127.0.0.1 - port: "5432" - user: postgres - password: "pass" -bigtable: - project: explorer - instance: explorer -eth1ErigonEndpoint: 'http://127.0.0.1:6001' -eth1GethEndpoint: 'http://127.0.0.1:6001' -redisCacheEndpoint: '127.0.0.1:6379' -tieredCacheProvider: 'redis' -#tieredCacheProvider: 'bigtable' -frontend: - enabled: true - siteDomain: "local-testnet.beaconcha.in" - siteName: 'Open Source Ethereum (ETH) Testnet Explorer' # Name of the site, displayed in the title tag - siteSubtitle: "Showing a local testnet." - server: - host: '0.0.0.0' # Address to listen on - port: '8080' # Port to listen on - readerDatabase: - name: db - host: 127.0.0.1 - port: "5432" - user: postgres - password: "pass" - writerDatabase: - name: db - host: 127.0.0.1 - port: "5432" - user: postgres - password: "pass" - sessionSecret: "11111111111111111111111111111111" - jwtSigningSecret: "1111111111111111111111111111111111111111111111111111111111111111" - jwtIssuer: "localhost" - jwtValidityInMinutes: 30 - maxMailsPerEmailPerDay: 10 - mail: - mailgun: - sender: no-reply@localhost - domain: mg.localhost - privateKey: "key-11111111111111111111111111111111" - csrfAuthKey: '1111111111111111111111111111111111111111111111111111111111111111' -indexer: - enabled: false - # fullIndexOnStartup: false # Perform a one time full db index on startup - # indexMissingEpochsOnStartup: true # Check for missing epochs and export them after startup - node: - host: 127.0.0.1 - port: '8001' - type: lighthouse - eth1Endpoint: "http://127.0.0.1:6001" - eth1DepositContractAddress: '0x4242424242424242424242424242424242424242' - eth1DepositContractFirstBlock: 0 \ No newline at end of file From db296f87d90feb18e19ee073e3cebdcfafa7cf5e Mon Sep 17 00:00:00 2001 From: peter <1674920+peterbitfly@users.noreply.github.com> Date: Thu, 31 Aug 2023 09:46:43 +0200 Subject: [PATCH 033/106] (BIDS-2379) update readme --- local-deployment/README.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/local-deployment/README.md b/local-deployment/README.md index 82c69a7dad..7e4b99ff9b 100644 --- a/local-deployment/README.md +++ b/local-deployment/README.md @@ -74,17 +74,17 @@ cp ~/eth2-beaconchain-explorer/local-deployment/testnet-config-example.yml ~/tes ``` # Initialize the db schema ``` -INDEXER_NODE_PORT= BIGTABLE_EMULATOR_HOST="127.0.0.1:9000" ~/eth2-beaconchain-explorer/bin/misc -config ~/eth2-beaconchain-explorer/local-deployment/testnet-config.yml -command applyDbSchema +BIGTABLE_EMULATOR_HOST="127.0.0.1:9000" ~/eth2-beaconchain-explorer/bin/misc -config ~/testnet/explorer-config.yml -command applyDbSchema ``` # Start the indexer ``` -BIGTABLE_EMULATOR_HOST="127.0.0.1:9000" ~/eth2-beaconchain-explorer/bin/explorer -config ~/eth2-beaconchain-explorer/local-deployment/testnet-config.yml +BIGTABLE_EMULATOR_HOST="127.0.0.1:9000" ~/eth2-beaconchain-explorer/bin/explorer -config ~/testnet/explorer-config.yml ``` # Start the frontend-data-updater ``` -BIGTABLE_EMULATOR_HOST="127.0.0.1:9000" ~/eth2-beaconchain-explorer/bin/frontend-data-updater -config ~/eth2-beaconchain-explorer/local-deployment/testnet-config.yml +BIGTABLE_EMULATOR_HOST="127.0.0.1:9000" ~/eth2-beaconchain-explorer/bin/frontend-data-updater -config ~/testnet/explorer-config.yml ``` # Start the frontend ``` -BIGTABLE_EMULATOR_HOST="127.0.0.1:9000" ~/eth2-beaconchain-explorer/bin/frontend-data-updater -config ~/eth2-beaconchain-explorer/local-deployment/testnet-config-frontend.yml +BIGTABLE_EMULATOR_HOST="127.0.0.1:9000" ~/eth2-beaconchain-explorer/bin/frontend-data-updater -config ~/testnet/explorer-config.yml ``` From 9dcd8c42ec141ec2fb9b3d02e2ec21e9b6844ddb Mon Sep 17 00:00:00 2001 From: peter <1674920+peterbitfly@users.noreply.github.com> Date: Thu, 31 Aug 2023 09:47:07 +0200 Subject: [PATCH 034/106] (BIDS-2379) adjust frontend start parameters --- local-deployment/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/local-deployment/README.md b/local-deployment/README.md index 7e4b99ff9b..8f97b8cb40 100644 --- a/local-deployment/README.md +++ b/local-deployment/README.md @@ -86,5 +86,5 @@ BIGTABLE_EMULATOR_HOST="127.0.0.1:9000" ~/eth2-beaconchain-explorer/bin/frontend ``` # Start the frontend ``` -BIGTABLE_EMULATOR_HOST="127.0.0.1:9000" ~/eth2-beaconchain-explorer/bin/frontend-data-updater -config ~/testnet/explorer-config.yml +INDEXER_ENABLED=false FRONTEND_ENABLED=true BIGTABLE_EMULATOR_HOST="127.0.0.1:9000" ~/eth2-beaconchain-explorer/bin/frontend-data-updater -config ~/testnet/explorer-config.yml ``` From 0c964015bd201cffe0c2764982443e359e1c8caa Mon Sep 17 00:00:00 2001 From: peter <1674920+peterbitfly@users.noreply.github.com> Date: Thu, 31 Aug 2023 10:04:58 +0200 Subject: [PATCH 035/106] (BIDS-2379) fix tracing of genesis block --- local-deployment/README.md | 2 +- local-deployment/network-params.json | 29 ++++++++++++++++++++++++++++ rpc/erigon.go | 3 +++ 3 files changed, 33 insertions(+), 1 deletion(-) create mode 100644 local-deployment/network-params.json diff --git a/local-deployment/README.md b/local-deployment/README.md index 8f97b8cb40..5c6abad5bf 100644 --- a/local-deployment/README.md +++ b/local-deployment/README.md @@ -65,7 +65,7 @@ bash ~/eth2-beaconchain-explorer/local-deployment/init-bigtable.sh ``` # Start up the local testnet nodes ``` -kurtosis run --enclave my-testnet github.com/kurtosis-tech/eth-network-package +kurtosis run --enclave my-testnet github.com/kurtosis-tech/eth-network-package "$(cat ~/eth2-beaconchain-explorer/local-deployment/network-params.json)" ``` Take note of the http API ports of the el & cl clients. # Copy the example config file and add the el & cl client ports diff --git a/local-deployment/network-params.json b/local-deployment/network-params.json new file mode 100644 index 0000000000..6c1b07d26b --- /dev/null +++ b/local-deployment/network-params.json @@ -0,0 +1,29 @@ +{ + "participants": [ + { + "el_client_type": "geth", + "el_client_image": "", + "el_client_log_level": "", + "cl_client_type": "lighthouse", + "cl_client_image": "", + "cl_client_log_level": "", + "beacon_extra_params": [], + "el_extra_params": ["--http.api='admin,eth,net,debug,txpool'"], + "validator_extra_params": [], + "builder_network_params": null, + "count": 1 + } + ], + "network_params": { + "preregistered_validator_keys_mnemonic": "giant issue aisle success illegal bike spike question tent bar rely arctic volcano long crawl hungry vocal artwork sniff fantasy very lucky have athlete", + "num_validator_keys_per_node": 64, + "network_id": "3151908", + "deposit_contract_address": "0x4242424242424242424242424242424242424242", + "seconds_per_slot": 12, + "genesis_delay": 120, + "capella_fork_epoch": 5, + "deneb_fork_epoch": 500 + }, + "global_client_log_level": "info" + } + \ No newline at end of file diff --git a/rpc/erigon.go b/rpc/erigon.go index 0b97ae5713..039824793b 100644 --- a/rpc/erigon.go +++ b/rpc/erigon.go @@ -191,6 +191,9 @@ func (client *ErigonClient) GetBlock(number int64) (*types.Eth1Block, *types.Get g := new(errgroup.Group) g.Go(func() error { + if block.NumberU64() == 0 { // genesis block is not traceable + return nil + } traces, err := client.TraceParity(block.NumberU64()) if err != nil { From 872bb2e36fda82a567d6b8ee8f63d454bc260625 Mon Sep 17 00:00:00 2001 From: peter <1674920+peterbitfly@users.noreply.github.com> Date: Thu, 31 Aug 2023 12:22:43 +0200 Subject: [PATCH 036/106] (BIDS-2379) simplify local deployment --- .gitignore | 1 + Dockerfile | 2 +- Makefile | 2 +- db/bigtable.go | 10 ++ local-deployment/README.md | 31 ++-- local-deployment/docker-compose.yml | 31 ---- local-deployment/kurtosis.yml | 1 + local-deployment/main.star | 51 +++++++ local-deployment/provision-explorer-config.sh | 144 ++++++++++++++++++ local-deployment/testnet-config-example.yml | 61 -------- types/config.go | 6 +- 11 files changed, 223 insertions(+), 117 deletions(-) delete mode 100644 local-deployment/docker-compose.yml create mode 100644 local-deployment/kurtosis.yml create mode 100644 local-deployment/main.star create mode 100644 local-deployment/provision-explorer-config.sh delete mode 100644 local-deployment/testnet-config-example.yml diff --git a/.gitignore b/.gitignore index 122ec2e3d2..49ff779018 100644 --- a/.gitignore +++ b/.gitignore @@ -29,3 +29,4 @@ lastAttestationCache cmd/forkwatch **/.DS_Store _gitignore/ +local-deployment/config.yml \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index e07628d2e8..fcc5b91110 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,5 @@ # The dockerfile is currently still WIP and might be broken -FROM golang:1.18 AS build-env +FROM golang:1.20 AS build-env COPY go.mod go.sum /src/ WORKDIR /src RUN go mod download diff --git a/Makefile b/Makefile index b110581b4e..ecf6128c05 100644 --- a/Makefile +++ b/Makefile @@ -5,7 +5,7 @@ BUILDDATE=`date -u +"%Y-%m-%dT%H:%M:%S%:z"` PACKAGE=eth2-exporter LDFLAGS="-X ${PACKAGE}/version.Version=${VERSION} -X ${PACKAGE}/version.BuildDate=${BUILDDATE} -X ${PACKAGE}/version.GitCommit=${GITCOMMIT} -X ${PACKAGE}/version.GitDate=${GITDATE} -s -w" -all: explorer stats frontend-data-updater eth1indexer ethstore-exporter rewards-exporter node-jobs-processor signatures notification-sender notification-collector +all: explorer stats frontend-data-updater eth1indexer ethstore-exporter rewards-exporter node-jobs-processor signatures notification-sender notification-collector misc lint: golint ./... diff --git a/db/bigtable.go b/db/bigtable.go index 459ce4d10a..e10eeda0b2 100644 --- a/db/bigtable.go +++ b/db/bigtable.go @@ -6,6 +6,7 @@ import ( "eth2-exporter/types" "eth2-exporter/utils" "fmt" + "os" "sort" "strconv" "strings" @@ -67,6 +68,15 @@ type Bigtable struct { } func InitBigtable(project, instance, chainId, redisAddress string) (*Bigtable, error) { + + if utils.Config.Bigtable.Emulator { + logger.Infof("using emulated local bigtable environment, setting BIGTABLE_EMULATOR_HOST env variable to 127.0.0.1:%d", utils.Config.Bigtable.EmulatorPort) + err := os.Setenv("BIGTABLE_EMULATOR_HOST", fmt.Sprintf("127.0.0.1:%d", utils.Config.Bigtable.EmulatorPort)) + + if err != nil { + logger.Fatalf("unable to set bigtable emulator environment variable: %v", err) + } + } ctx, cancel := context.WithTimeout(context.Background(), time.Second*30) defer cancel() diff --git a/local-deployment/README.md b/local-deployment/README.md index 5c6abad5bf..6d75698c2b 100644 --- a/local-deployment/README.md +++ b/local-deployment/README.md @@ -53,38 +53,27 @@ sudo apt install build-essential make all make misc ``` -# Start postgres, redis & little_bigtable +# Start postgres, redis, little_bigtable & the eth test network ``` cd ~/eth2-beaconchain-explorer/local-deployment/ -docker compose up -d +kurtosis clean -a && kurtosis run --enclave my-testnet . "$(cat network-params.json)" ``` -Redis will be available on port 6379, little_bigtable on port 9000 and postgres on port 5432 (username `postgres`, password `pass`, db `db`) -# Initialize the bigtable schema +# Generate the explorer config file for the deployed testnet ``` -bash ~/eth2-beaconchain-explorer/local-deployment/init-bigtable.sh -``` -# Start up the local testnet nodes -``` -kurtosis run --enclave my-testnet github.com/kurtosis-tech/eth-network-package "$(cat ~/eth2-beaconchain-explorer/local-deployment/network-params.json)" -``` -Take note of the http API ports of the el & cl clients. -# Copy the example config file and add the el & cl client ports -``` -cp ~/eth2-beaconchain-explorer/local-deployment/testnet-config-example.yml ~/testnet/explorer-config.yml -``` -# Initialize the db schema -``` -BIGTABLE_EMULATOR_HOST="127.0.0.1:9000" ~/eth2-beaconchain-explorer/bin/misc -config ~/testnet/explorer-config.yml -command applyDbSchema +cd ~/eth2-beaconchain-explorer/local-deployment/ +bash provision-explorer-config.sh ``` +This will generate a config.yml to be used by the explorer and then create the bigtable & postgres schema + # Start the indexer ``` -BIGTABLE_EMULATOR_HOST="127.0.0.1:9000" ~/eth2-beaconchain-explorer/bin/explorer -config ~/testnet/explorer-config.yml +INDEXER_ENABLED=true ~/eth2-beaconchain-explorer/bin/explorer -config ~/eth2-beaconchain-explorer/local-deployment/config.yml ``` # Start the frontend-data-updater ``` -BIGTABLE_EMULATOR_HOST="127.0.0.1:9000" ~/eth2-beaconchain-explorer/bin/frontend-data-updater -config ~/testnet/explorer-config.yml +~/eth2-beaconchain-explorer/bin/frontend-data-updater -config ~/eth2-beaconchain-explorer/local-deployment/config.yml ``` # Start the frontend ``` -INDEXER_ENABLED=false FRONTEND_ENABLED=true BIGTABLE_EMULATOR_HOST="127.0.0.1:9000" ~/eth2-beaconchain-explorer/bin/frontend-data-updater -config ~/testnet/explorer-config.yml +FRONTEND_ENABLED=true ~/eth2-beaconchain-explorer/bin/frontend-data-updater -config ~/eth2-beaconchain-explorer/local-deployment/config.yml ``` diff --git a/local-deployment/docker-compose.yml b/local-deployment/docker-compose.yml deleted file mode 100644 index a62eec5a87..0000000000 --- a/local-deployment/docker-compose.yml +++ /dev/null @@ -1,31 +0,0 @@ -version: '3' -services: - ################ dbs - redis: - image: redis:7 - restart: unless-stopped - volumes: - - ./docker-volumes/redis:/data - ports: - - 6379:6379 - postgres: - image: postgres:14 - restart: unless-stopped - environment: - - POSTGRES_PASSWORD=pass - - POSTGRES_USER=postgres - - POSTGRES_DB=db - - PGDATA=/var/lib/postgresql/data - volumes: - - ./docker-volumes/postgres:/var/lib/postgresql/data - - ./tables.sql:/docker-entrypoint-initdb.d/init.sql - ports: - - 5432:5432 - little-bigtable: - image: gobitfly/little_bigtable - restart: unless-stopped - volumes: - - ./docker-volumes/little_bigtable:/app/data - ports: - - 9000:9000 - diff --git a/local-deployment/kurtosis.yml b/local-deployment/kurtosis.yml new file mode 100644 index 0000000000..b9e2c298c7 --- /dev/null +++ b/local-deployment/kurtosis.yml @@ -0,0 +1 @@ +name: "github.com/gobitfly/eth2-beaconchain-explorer/local-testnet" \ No newline at end of file diff --git a/local-deployment/main.star b/local-deployment/main.star new file mode 100644 index 0000000000..fcd39d148b --- /dev/null +++ b/local-deployment/main.star @@ -0,0 +1,51 @@ +parse_input = import_module("github.com/kurtosis-tech/eth2-package/src/package_io/parse_input.star") +eth_network_module = import_module("github.com/kurtosis-tech/eth-network-package/main.star") + +POSTGRES_PORT_ID = "postgres" +POSTGRES_DB = "db" +POSTGRES_USER = "postgres" +POSTGRES_PASSWORD = "pass" + +REDIS_PORT_ID = "redis" + +LITTLE_BIGTABLE_PORT_ID = "littlebigtable" + +def run(plan, args): + # Add a Postgres server + postgres = plan.add_service( + name = "postgres", + config = ServiceConfig( + image = "postgres:15.2-alpine", + ports = { + POSTGRES_PORT_ID: PortSpec(5432, application_protocol = "postgresql"), + }, + env_vars = { + "POSTGRES_DB": POSTGRES_DB, + "POSTGRES_USER": POSTGRES_USER, + "POSTGRES_PASSWORD": POSTGRES_PASSWORD, + }, + ), + ) + # Add a redis server + redis = plan.add_service( + name = "redis", + config = ServiceConfig( + image = "redis:7", + ports = { + REDIS_PORT_ID: PortSpec(6379, application_protocol = "tcp"), + }, + ), + ) + # Add a little bigtable server + littlebigtable = plan.add_service( + name = "littlebigtable", + config = ServiceConfig( + image = "gobitfly/little_bigtable:latest", + ports = { + LITTLE_BIGTABLE_PORT_ID: PortSpec(9000, application_protocol = "tcp"), + }, + ), + ) + + eth_network_participants, cl_genesis_timestamp, cl_genesis_root_hash = eth_network_module.run(plan, args) + diff --git a/local-deployment/provision-explorer-config.sh b/local-deployment/provision-explorer-config.sh new file mode 100644 index 0000000000..08e42f4e9e --- /dev/null +++ b/local-deployment/provision-explorer-config.sh @@ -0,0 +1,144 @@ +#! /bin/bash +CL_PORT=$(kurtosis enclave inspect my-testnet | grep 4000/tcp | tr -s ' ' | cut -d " " -f 6 | sed -e 's/http\:\/\/127.0.0.1\://' | grep "\S") +echo "CL Node port is $CL_PORT" + +EL_PORT=$(kurtosis enclave inspect my-testnet | grep 8545/tcp | tr -s ' ' | cut -d " " -f 5 | sed -e 's/127.0.0.1\://' | grep "\S") +echo "EL Node port is $EL_PORT" + +REDIS_PORT=$(kurtosis enclave inspect my-testnet | grep 6379/tcp | tr -s ' ' | cut -d " " -f 6 | sed -e 's/tcp\:\/\/127.0.0.1\://' | grep "\S") +echo "Redis port is $REDIS_PORT" + +POSTGRES_PORT=$(kurtosis enclave inspect my-testnet | grep 5432/tcp | tr -s ' ' | cut -d " " -f 6 | sed -e 's/postgresql\:\/\/127.0.0.1\://' | grep "\S") +echo "Postgres port is $POSTGRES_PORT" + +LBT_PORT=$(kurtosis enclave inspect my-testnet | grep 9000/tcp | tr -s ' ' | cut -d " " -f 6 | sed -e 's/tcp\:\/\/127.0.0.1\://' | grep "\S") +echo "Little bigtable port is $LBT_PORT" + +cat >config.yml <' -eth1GethEndpoint: 'http://127.0.0.1:' -redisCacheEndpoint: '127.0.0.1:6379' -tieredCacheProvider: 'redis' -#tieredCacheProvider: 'bigtable' -frontend: - siteDomain: "local-testnet.beaconcha.in" - siteName: 'Open Source Ethereum (ETH) Testnet Explorer' # Name of the site, displayed in the title tag - siteSubtitle: "Showing a local testnet." - server: - host: '0.0.0.0' # Address to listen on - port: '8080' # Port to listen on - readerDatabase: - name: db - host: 127.0.0.1 - port: "5432" - user: postgres - password: "pass" - writerDatabase: - name: db - host: 127.0.0.1 - port: "5432" - user: postgres - password: "pass" - sessionSecret: "11111111111111111111111111111111" - jwtSigningSecret: "1111111111111111111111111111111111111111111111111111111111111111" - jwtIssuer: "localhost" - jwtValidityInMinutes: 30 - maxMailsPerEmailPerDay: 10 - mail: - mailgun: - sender: no-reply@localhost - domain: mg.localhost - privateKey: "key-11111111111111111111111111111111" - csrfAuthKey: '1111111111111111111111111111111111111111111111111111111111111111' -indexer: - enabled: true - # fullIndexOnStartup: false # Perform a one time full db index on startup - # indexMissingEpochsOnStartup: true # Check for missing epochs and export them after startup - node: - host: 127.0.0.1 - port: '' - type: lighthouse - eth1DepositContractFirstBlock: 0 \ No newline at end of file diff --git a/types/config.go b/types/config.go index 1aede97a0c..c45f0e689c 100644 --- a/types/config.go +++ b/types/config.go @@ -26,8 +26,10 @@ type Config struct { MaxIdleConns int `yaml:"maxIdleConns" envconfig:"WRITER_DB_MAX_IDLE_CONNS"` } `yaml:"writerDatabase"` Bigtable struct { - Project string `yaml:"project" envconfig:"BIGTABLE_PROJECT"` - Instance string `yaml:"instance" envconfig:"BIGTABLE_INSTANCE"` + Project string `yaml:"project" envconfig:"BIGTABLE_PROJECT"` + Instance string `yaml:"instance" envconfig:"BIGTABLE_INSTANCE"` + Emulator bool `yaml:"emulator" envconfig:"BIGTABLE_EMULATOR"` + EmulatorPort int `yaml:"emulatorPort" envconfig:"BIGTABLE_EMULATOR_PORT"` } `yaml:"bigtable"` Chain struct { Name string `yaml:"name" envconfig:"CHAIN_NAME"` From 5f56e4c467dfdf87416bb50b42ef59d1ceb34d84 Mon Sep 17 00:00:00 2001 From: peter <1674920+peterbitfly@users.noreply.github.com> Date: Thu, 31 Aug 2023 12:38:59 +0200 Subject: [PATCH 037/106] (BIDS-2379) update config --- local-deployment/README.md | 3 +-- local-deployment/network-params.json | 4 ++-- local-deployment/provision-explorer-config.sh | 1 - 3 files changed, 3 insertions(+), 5 deletions(-) diff --git a/local-deployment/README.md b/local-deployment/README.md index 6d75698c2b..729de4a56e 100644 --- a/local-deployment/README.md +++ b/local-deployment/README.md @@ -51,7 +51,6 @@ cd eth2-beaconchain-explorer ``` sudo apt install build-essential make all -make misc ``` # Start postgres, redis, little_bigtable & the eth test network ``` @@ -67,7 +66,7 @@ This will generate a config.yml to be used by the explorer and then create the b # Start the indexer ``` -INDEXER_ENABLED=true ~/eth2-beaconchain-explorer/bin/explorer -config ~/eth2-beaconchain-explorer/local-deployment/config.yml +INDEXER_ENABLED=true ~/eth2-beaconchain-explorer/bin/explorer -config ~/eth2-beaconchain-explorer/local-deployment/config.yml ``` # Start the frontend-data-updater ``` diff --git a/local-deployment/network-params.json b/local-deployment/network-params.json index 6c1b07d26b..74e7f911fb 100644 --- a/local-deployment/network-params.json +++ b/local-deployment/network-params.json @@ -19,8 +19,8 @@ "num_validator_keys_per_node": 64, "network_id": "3151908", "deposit_contract_address": "0x4242424242424242424242424242424242424242", - "seconds_per_slot": 12, - "genesis_delay": 120, + "seconds_per_slot": 4, + "genesis_delay": 10, "capella_fork_epoch": 5, "deneb_fork_epoch": 500 }, diff --git a/local-deployment/provision-explorer-config.sh b/local-deployment/provision-explorer-config.sh index 08e42f4e9e..d422839baa 100644 --- a/local-deployment/provision-explorer-config.sh +++ b/local-deployment/provision-explorer-config.sh @@ -69,7 +69,6 @@ frontend: privateKey: "key-11111111111111111111111111111111" csrfAuthKey: '1111111111111111111111111111111111111111111111111111111111111111' indexer: - enabled: true # fullIndexOnStartup: false # Perform a one time full db index on startup # indexMissingEpochsOnStartup: true # Check for missing epochs and export them after startup node: From b4b6179a6bd53e54eb3932c7b6b4431355c9d488 Mon Sep 17 00:00:00 2001 From: peter <1674920+peterbitfly@users.noreply.github.com> Date: Thu, 31 Aug 2023 12:49:31 +0200 Subject: [PATCH 038/106] (BIDS-2379) add tx spammer to local depyloment --- local-deployment/main.star | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/local-deployment/main.star b/local-deployment/main.star index fcd39d148b..396d77bdf0 100644 --- a/local-deployment/main.star +++ b/local-deployment/main.star @@ -1,5 +1,7 @@ parse_input = import_module("github.com/kurtosis-tech/eth2-package/src/package_io/parse_input.star") eth_network_module = import_module("github.com/kurtosis-tech/eth-network-package/main.star") +transaction_spammer = import_module("github.com/kurtosis-tech/eth2-package/src/transaction_spammer/transaction_spammer.star") +genesis_constants = import_module("github.com/kurtosis-tech/eth-network-package/src/prelaunch_data_generator/genesis_constants/genesis_constants.star") POSTGRES_PORT_ID = "postgres" POSTGRES_DB = "db" @@ -47,5 +49,15 @@ def run(plan, args): ), ) - eth_network_participants, cl_genesis_timestamp, cl_genesis_root_hash = eth_network_module.run(plan, args) + all_participants, cl_genesis_timestamp, genesis_validators_root = eth_network_module.run(plan, args) + + all_el_client_contexts = [] + all_cl_client_contexts = [] + for participant in all_participants: + all_el_client_contexts.append(participant.el_client_context) + all_cl_client_contexts.append(participant.cl_client_context) + + plan.print("Launching transaction spammer") + transaction_spammer.launch_transaction_spammer(plan, genesis_constants.PRE_FUNDED_ACCOUNTS, all_el_client_contexts[0]) + plan.print("Succesfully launched transaction spammer") From c58324f01e7915882bd44de957e896f10f93563d Mon Sep 17 00:00:00 2001 From: peter <1674920+peterbitfly@users.noreply.github.com> Date: Thu, 31 Aug 2023 12:51:20 +0200 Subject: [PATCH 039/106] (BIDS-2379) remove obsole bigtable init file --- local-deployment/init-bigtable.sh | 56 ------------------------------- 1 file changed, 56 deletions(-) delete mode 100644 local-deployment/init-bigtable.sh diff --git a/local-deployment/init-bigtable.sh b/local-deployment/init-bigtable.sh deleted file mode 100644 index 824ddb8d73..0000000000 --- a/local-deployment/init-bigtable.sh +++ /dev/null @@ -1,56 +0,0 @@ -#!/bin/bash - -PROJECT="explorer" -INSTANCE="explorer" -HOST="127.0.0.1:9000" - -BIGTABLE_EMULATOR_HOST=$HOST cbt --project $PROJECT --instance $INSTANCE createtable beaconchain_validator_balances -BIGTABLE_EMULATOR_HOST=$HOST cbt --project $PROJECT --instance $INSTANCE createfamily beaconchain_validator_balances vb -BIGTABLE_EMULATOR_HOST=$HOST cbt --project $PROJECT --instance $INSTANCE createtable beaconchain_validator_attestations -BIGTABLE_EMULATOR_HOST=$HOST cbt --project $PROJECT --instance $INSTANCE createfamily beaconchain_validator_attestations at -BIGTABLE_EMULATOR_HOST=$HOST cbt --project $PROJECT --instance $INSTANCE createtable beaconchain_validator_proposals -BIGTABLE_EMULATOR_HOST=$HOST cbt --project $PROJECT --instance $INSTANCE createfamily beaconchain_validator_proposals pr -BIGTABLE_EMULATOR_HOST=$HOST cbt --project $PROJECT --instance $INSTANCE createtable beaconchain_validator_sync -BIGTABLE_EMULATOR_HOST=$HOST cbt --project $PROJECT --instance $INSTANCE createfamily beaconchain_validator_sync sc -BIGTABLE_EMULATOR_HOST=$HOST cbt --project $PROJECT --instance $INSTANCE createtable beaconchain_validator_income -BIGTABLE_EMULATOR_HOST=$HOST cbt --project $PROJECT --instance $INSTANCE createfamily beaconchain_validator_income id -BIGTABLE_EMULATOR_HOST=$HOST cbt --project $PROJECT --instance $INSTANCE createfamily beaconchain_validator_income stats -BIGTABLE_EMULATOR_HOST=$HOST cbt --project $PROJECT --instance $INSTANCE createtable beaconchain_validators -BIGTABLE_EMULATOR_HOST=$HOST cbt --project $PROJECT --instance $INSTANCE createfamily beaconchain_validators at - -BIGTABLE_EMULATOR_HOST=$HOST cbt --project $PROJECT --instance $INSTANCE setgcpolicy beaconchain_validators at maxversions=1 -BIGTABLE_EMULATOR_HOST=$HOST cbt --project $PROJECT --instance $INSTANCE createtable blocks -BIGTABLE_EMULATOR_HOST=$HOST cbt --project $PROJECT --instance $INSTANCE createfamily blocks default - -BIGTABLE_EMULATOR_HOST=$HOST cbt --project $PROJECT --instance $INSTANCE setgcpolicy blocks default maxversions=1 -BIGTABLE_EMULATOR_HOST=$HOST cbt --project $PROJECT --instance $INSTANCE createtable cache -BIGTABLE_EMULATOR_HOST=$HOST cbt --project $PROJECT --instance $INSTANCE createfamily cache 10_min -BIGTABLE_EMULATOR_HOST=$HOST cbt --project $PROJECT --instance $INSTANCE createfamily cache 1_day -BIGTABLE_EMULATOR_HOST=$HOST cbt --project $PROJECT --instance $INSTANCE createfamily cache 1_hour - -BIGTABLE_EMULATOR_HOST=$HOST cbt --project $PROJECT --instance $INSTANCE setgcpolicy cache 10_min maxage=10m and maxversions=1 -BIGTABLE_EMULATOR_HOST=$HOST cbt --project $PROJECT --instance $INSTANCE setgcpolicy cache 1_day maxage=1d and maxversions=1 -BIGTABLE_EMULATOR_HOST=$HOST cbt --project $PROJECT --instance $INSTANCE setgcpolicy cache 1_hour maxage=1h and maxversions=1 -BIGTABLE_EMULATOR_HOST=$HOST cbt --project $PROJECT --instance $INSTANCE createtable data -BIGTABLE_EMULATOR_HOST=$HOST cbt --project $PROJECT --instance $INSTANCE createfamily data c -BIGTABLE_EMULATOR_HOST=$HOST cbt --project $PROJECT --instance $INSTANCE createfamily data f - -BIGTABLE_EMULATOR_HOST=$HOST cbt --project $PROJECT --instance $INSTANCE setgcpolicy data c maxage=1d -BIGTABLE_EMULATOR_HOST=$HOST cbt --project $PROJECT --instance $INSTANCE createtable machine_metrics -BIGTABLE_EMULATOR_HOST=$HOST cbt --project $PROJECT --instance $INSTANCE createfamily machine_metrics mm - -BIGTABLE_EMULATOR_HOST=$HOST cbt --project $PROJECT --instance $INSTANCE setgcpolicy machine_metrics mm maxage=31d -BIGTABLE_EMULATOR_HOST=$HOST cbt --project $PROJECT --instance $INSTANCE createtable metadata -BIGTABLE_EMULATOR_HOST=$HOST cbt --project $PROJECT --instance $INSTANCE createfamily metadata a -BIGTABLE_EMULATOR_HOST=$HOST cbt --project $PROJECT --instance $INSTANCE createfamily metadata c -BIGTABLE_EMULATOR_HOST=$HOST cbt --project $PROJECT --instance $INSTANCE createfamily metadata erc1155 -BIGTABLE_EMULATOR_HOST=$HOST cbt --project $PROJECT --instance $INSTANCE createfamily metadata erc20 -BIGTABLE_EMULATOR_HOST=$HOST cbt --project $PROJECT --instance $INSTANCE createfamily metadata erc721 -BIGTABLE_EMULATOR_HOST=$HOST cbt --project $PROJECT --instance $INSTANCE createfamily metadata series - -BIGTABLE_EMULATOR_HOST=$HOST cbt --project $PROJECT --instance $INSTANCE setgcpolicy metadata series maxversions=1 -BIGTABLE_EMULATOR_HOST=$HOST cbt --project $PROJECT --instance $INSTANCE createtable metadata_updates -BIGTABLE_EMULATOR_HOST=$HOST cbt --project $PROJECT --instance $INSTANCE createfamily metadata_updates blocks -BIGTABLE_EMULATOR_HOST=$HOST cbt --project $PROJECT --instance $INSTANCE createfamily metadata_updates f - -BIGTABLE_EMULATOR_HOST=$HOST cbt --project $PROJECT --instance $INSTANCE setgcpolicy metadata_updates blocks maxage=1d \ No newline at end of file From 61063f6e0d60de1f40b787b6088659aa18a8161e Mon Sep 17 00:00:00 2001 From: peter <1674920+peterbitfly@users.noreply.github.com> Date: Thu, 31 Aug 2023 12:55:05 +0200 Subject: [PATCH 040/106] (BIDS-2379) fix genesis balance export --- db/db.go | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/db/db.go b/db/db.go index 9afe9a0c1e..b1a9aeb8eb 100644 --- a/db/db.go +++ b/db/db.go @@ -1022,7 +1022,13 @@ func saveValidators(data *types.EpochData, tx *sqlx.Tx, client rpc.Client) error if data.Epoch == 0 { var err error - genesisBalances, err = BigtableClient.GetValidatorBalanceHistory([]uint64{}, 0, 0) + + indices := make([]uint64, 0, len(data.Validators)) + + for _, validator := range data.Validators { + indices = append(indices, validator.Index) + } + genesisBalances, err = BigtableClient.GetValidatorBalanceHistory(indices, 0, 0) if err != nil { return err } From 2fba664d0ecf38f8e794cc00d2b169ec412d086a Mon Sep 17 00:00:00 2001 From: peter <1674920+peterbitfly@users.noreply.github.com> Date: Thu, 31 Aug 2023 13:03:59 +0200 Subject: [PATCH 041/106] (BIDS-2379) eth1index to use config node url as fallback --- cmd/eth1indexer/main.go | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/cmd/eth1indexer/main.go b/cmd/eth1indexer/main.go index 376c87c76b..01197d4b79 100644 --- a/cmd/eth1indexer/main.go +++ b/cmd/eth1indexer/main.go @@ -112,7 +112,15 @@ func main() { defer db.WriterDb.Close() if erigonEndpoint == nil || *erigonEndpoint == "" { - utils.LogFatal(nil, "no erigon node url provided", 0) + + if utils.Config.Eth1ErigonEndpoint == "" { + + utils.LogFatal(nil, "no erigon node url provided", 0) + } else { + logrus.Info("applying erigon endpoint from config") + *erigonEndpoint = utils.Config.Eth1ErigonEndpoint + } + } logrus.Infof("using erigon node at %v", *erigonEndpoint) From 8fc35601394414b074d6018b0e2aec238fe74e37 Mon Sep 17 00:00:00 2001 From: peter <1674920+peterbitfly@users.noreply.github.com> Date: Thu, 31 Aug 2023 13:23:03 +0200 Subject: [PATCH 042/106] (BIDS-2379) fix parsing of geth style traces --- rpc/erigon.go | 19 +++++++------------ 1 file changed, 7 insertions(+), 12 deletions(-) diff --git a/rpc/erigon.go b/rpc/erigon.go index 039824793b..32a19fd742 100644 --- a/rpc/erigon.go +++ b/rpc/erigon.go @@ -371,6 +371,10 @@ func (client *ErigonClient) GetLatestEth1BlockNumber() (uint64, error) { return latestBlock.NumberU64(), nil } +type GethTraceCallResultWrapper struct { + Result *GethTraceCallResult +} + type GethTraceCallResult struct { TransactionPosition int Time string @@ -386,15 +390,6 @@ type GethTraceCallResult struct { Calls []*GethTraceCallResult } -type GethTraceCallData struct { - From common.Address - To common.Address - Gas hexutil.Uint64 - GasPrice hexutil.Big - Value hexutil.Big - Data hexutil.Bytes -} - var gethTracerArg = map[string]string{ "tracer": "callTracer", } @@ -415,7 +410,7 @@ func extractCalls(r *GethTraceCallResult, d *[]*GethTraceCallResult) { } func (client *ErigonClient) TraceGeth(blockHash common.Hash) ([]*GethTraceCallResult, error) { - var res []*GethTraceCallResult + var res []*GethTraceCallResultWrapper err := client.rpcClient.Call(&res, "debug_traceBlockByHash", blockHash, gethTracerArg) if err != nil { @@ -424,8 +419,8 @@ func (client *ErigonClient) TraceGeth(blockHash common.Hash) ([]*GethTraceCallRe data := make([]*GethTraceCallResult, 0, 20) for i, r := range res { - r.TransactionPosition = i - extractCalls(r, &data) + r.Result.TransactionPosition = i + extractCalls(r.Result, &data) } return data, nil From 2ad1b6d6763c507bf3cc92af2de53993501679a1 Mon Sep 17 00:00:00 2001 From: peter <1674920+peterbitfly@users.noreply.github.com> Date: Thu, 31 Aug 2023 13:29:13 +0200 Subject: [PATCH 043/106] (BIDS-2379) configure geth as archive node for local deployments --- local-deployment/network-params.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/local-deployment/network-params.json b/local-deployment/network-params.json index 74e7f911fb..2c7231f300 100644 --- a/local-deployment/network-params.json +++ b/local-deployment/network-params.json @@ -8,7 +8,7 @@ "cl_client_image": "", "cl_client_log_level": "", "beacon_extra_params": [], - "el_extra_params": ["--http.api='admin,eth,net,debug,txpool'"], + "el_extra_params": ["--http.api='admin,eth,net,debug,txpool' --syncmode full --gcmode archive"], "validator_extra_params": [], "builder_network_params": null, "count": 1 From 5b76b85db080f386d2d191e01c54061f5ad7ebf2 Mon Sep 17 00:00:00 2001 From: peter <1674920+peterbitfly@users.noreply.github.com> Date: Thu, 31 Aug 2023 13:36:41 +0200 Subject: [PATCH 044/106] (BIDS-2379) add eth1indexer instructions --- local-deployment/README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/local-deployment/README.md b/local-deployment/README.md index 729de4a56e..8487893683 100644 --- a/local-deployment/README.md +++ b/local-deployment/README.md @@ -72,6 +72,10 @@ INDEXER_ENABLED=true ~/eth2-beaconchain-explorer/bin/explorer -config ~/eth2-bea ``` ~/eth2-beaconchain-explorer/bin/frontend-data-updater -config ~/eth2-beaconchain-explorer/local-deployment/config.yml ``` +# Start the eth1indexer +``` +~/eth2-beaconchain-explorer/bin/eth1indexer --config ~/eth2-beaconchain-explorer/local-deployment/config.yml -blocks.concurrency 1 --balances.enabled +``` # Start the frontend ``` FRONTEND_ENABLED=true ~/eth2-beaconchain-explorer/bin/frontend-data-updater -config ~/eth2-beaconchain-explorer/local-deployment/config.yml From 69c80548a54848c6e390465025cae44cd680e99a Mon Sep 17 00:00:00 2001 From: peter <1674920+peterbitfly@users.noreply.github.com> Date: Thu, 31 Aug 2023 13:56:57 +0200 Subject: [PATCH 045/106] (BIDS-2379) fix frontend start command --- local-deployment/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/local-deployment/README.md b/local-deployment/README.md index 8487893683..5c612b2ecf 100644 --- a/local-deployment/README.md +++ b/local-deployment/README.md @@ -78,5 +78,5 @@ INDEXER_ENABLED=true ~/eth2-beaconchain-explorer/bin/explorer -config ~/eth2-bea ``` # Start the frontend ``` -FRONTEND_ENABLED=true ~/eth2-beaconchain-explorer/bin/frontend-data-updater -config ~/eth2-beaconchain-explorer/local-deployment/config.yml +FRONTEND_ENABLED=true ~/eth2-beaconchain-explorer/bin/explorer -config ~/eth2-beaconchain-explorer/local-deployment/config.yml ``` From bb7ec47d815e37ee0bcbd8468f51a1f8a5a8a8a6 Mon Sep 17 00:00:00 2001 From: peter <1674920+peterbitfly@users.noreply.github.com> Date: Thu, 31 Aug 2023 14:27:16 +0200 Subject: [PATCH 046/106] (BIDS-2379) simplify service startup --- local-deployment/README.md | 18 ++++------------- local-deployment/docker-compose.yml | 30 +++++++++++++++++++++++++++++ 2 files changed, 34 insertions(+), 14 deletions(-) create mode 100644 local-deployment/docker-compose.yml diff --git a/local-deployment/README.md b/local-deployment/README.md index 5c612b2ecf..b9ea346c32 100644 --- a/local-deployment/README.md +++ b/local-deployment/README.md @@ -64,19 +64,9 @@ bash provision-explorer-config.sh ``` This will generate a config.yml to be used by the explorer and then create the bigtable & postgres schema -# Start the indexer +# Start the explorer modules ``` -INDEXER_ENABLED=true ~/eth2-beaconchain-explorer/bin/explorer -config ~/eth2-beaconchain-explorer/local-deployment/config.yml -``` -# Start the frontend-data-updater -``` -~/eth2-beaconchain-explorer/bin/frontend-data-updater -config ~/eth2-beaconchain-explorer/local-deployment/config.yml -``` -# Start the eth1indexer -``` -~/eth2-beaconchain-explorer/bin/eth1indexer --config ~/eth2-beaconchain-explorer/local-deployment/config.yml -blocks.concurrency 1 --balances.enabled -``` -# Start the frontend -``` -FRONTEND_ENABLED=true ~/eth2-beaconchain-explorer/bin/explorer -config ~/eth2-beaconchain-explorer/local-deployment/config.yml +cd ~/eth2-beaconchain-explorer/local-deployment/ +docker-compose up -d ``` +You can then start / stop the exporter submodules using `docker-compose` diff --git a/local-deployment/docker-compose.yml b/local-deployment/docker-compose.yml new file mode 100644 index 0000000000..fc2970a929 --- /dev/null +++ b/local-deployment/docker-compose.yml @@ -0,0 +1,30 @@ +version: '3' +services: + indexer: + build: ../ + command: ./explorer -config /app/config.yml + volumes: + - ./config.yml:/app/config.yml + environment: + - INDEXER_ENABLED=true + network_mode: "host" + eth1indexer: + build: ../ + command: ./eth1indexer -config /app/config.yml -blocks.concurrency 1 -data.concurrency 1 --balances.enabled + volumes: + - ./config.yml:/app/config.yml + network_mode: "host" + frontend-data-updater: + build: ../ + command: ./frontend-data-updater -config /app/config.yml + volumes: + - ./config.yml:/app/config.yml + network_mode: "host" + frontend: + build: ../ + command: ./explorer -config /app/config.yml + volumes: + - ./config.yml:/app/config.yml + environment: + - FRONTEND_ENABLED=true + network_mode: "host" \ No newline at end of file From fcdf150843ce95ef3c47f5c5faced202a6470d6d Mon Sep 17 00:00:00 2001 From: peter <1674920+peterbitfly@users.noreply.github.com> Date: Thu, 31 Aug 2023 14:33:09 +0200 Subject: [PATCH 047/106] (BIDS-2379) implement fallback nodes for rewards exporter, add rewards exporter to local deployment script --- cmd/rewards-exporter/main.go | 18 ++++++++++++++++++ local-deployment/docker-compose.yml | 6 ++++++ 2 files changed, 24 insertions(+) diff --git a/cmd/rewards-exporter/main.go b/cmd/rewards-exporter/main.go index eeb0ec82f9..ed618fe244 100644 --- a/cmd/rewards-exporter/main.go +++ b/cmd/rewards-exporter/main.go @@ -65,6 +65,24 @@ func main() { defer db.ReaderDb.Close() defer db.WriterDb.Close() + if bnAddress == nil || *bnAddress == "" { + if utils.Config.Indexer.Node.Host == "" { + utils.LogFatal(nil, "no beacon node url provided", 0) + } else { + logrus.Info("applying becon node endpoint from config") + *bnAddress = fmt.Sprintf("http://%s:%s", utils.Config.Indexer.Node.Host, utils.Config.Indexer.Node.Port) + } + } + + if enAddress == nil || *enAddress == "" { + if utils.Config.Eth1ErigonEndpoint == "" { + utils.LogFatal(nil, "no execution node url provided", 0) + } else { + logrus.Info("applying execution node endpoint from config") + *enAddress = utils.Config.Eth1ErigonEndpoint + } + } + client := beacon.NewClient(*bnAddress, time.Minute*5) bt, err := db.InitBigtable(utils.Config.Bigtable.Project, utils.Config.Bigtable.Instance, fmt.Sprintf("%d", utils.Config.Chain.Config.DepositChainID), utils.Config.RedisCacheEndpoint) diff --git a/local-deployment/docker-compose.yml b/local-deployment/docker-compose.yml index fc2970a929..7d5c803177 100644 --- a/local-deployment/docker-compose.yml +++ b/local-deployment/docker-compose.yml @@ -14,6 +14,12 @@ services: volumes: - ./config.yml:/app/config.yml network_mode: "host" + rewards-exporter: + build: ../ + command: ./rewards-exporter -config /app/config.yml + volumes: + - ./config.yml:/app/config.yml + network_mode: "host" frontend-data-updater: build: ../ command: ./frontend-data-updater -config /app/config.yml From 08870dfea247611724f07b63f18a360b84815e97 Mon Sep 17 00:00:00 2001 From: peter <1674920+peterbitfly@users.noreply.github.com> Date: Thu, 31 Aug 2023 14:36:32 +0200 Subject: [PATCH 048/106] (BIDS-2379) update local deployment guide --- local-deployment/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/local-deployment/README.md b/local-deployment/README.md index b9ea346c32..ec467dd5c8 100644 --- a/local-deployment/README.md +++ b/local-deployment/README.md @@ -69,4 +69,4 @@ This will generate a config.yml to be used by the explorer and then create the b cd ~/eth2-beaconchain-explorer/local-deployment/ docker-compose up -d ``` -You can then start / stop the exporter submodules using `docker-compose` +You can start / stop the exporter submodules using `docker-compose` From 11e9a874c6bd5b3e6a6dde19bbe539840677eb48 Mon Sep 17 00:00:00 2001 From: peter <1674920+peterbitfly@users.noreply.github.com> Date: Thu, 31 Aug 2023 14:39:37 +0200 Subject: [PATCH 049/106] (BIDS-2379) add statistics to docker compose --- local-deployment/docker-compose.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/local-deployment/docker-compose.yml b/local-deployment/docker-compose.yml index 7d5c803177..04b964691f 100644 --- a/local-deployment/docker-compose.yml +++ b/local-deployment/docker-compose.yml @@ -20,6 +20,12 @@ services: volumes: - ./config.yml:/app/config.yml network_mode: "host" + statistics: + build: ../ + command: ./statistics -config /app/config.yml --charts.enabled --graffiti.enabled -validators.enabled + volumes: + - ./config.yml:/app/config.yml + network_mode: "host" frontend-data-updater: build: ../ command: ./frontend-data-updater -config /app/config.yml From 71a38c282c185367052e8ecf6346e05b18928800 Mon Sep 17 00:00:00 2001 From: peter <1674920+peterbitfly@users.noreply.github.com> Date: Thu, 31 Aug 2023 14:48:33 +0200 Subject: [PATCH 050/106] (BIDS-2379) improved config --- local-deployment/provision-explorer-config.sh | 2 +- rpc/erigon.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/local-deployment/provision-explorer-config.sh b/local-deployment/provision-explorer-config.sh index d422839baa..613c0c422b 100644 --- a/local-deployment/provision-explorer-config.sh +++ b/local-deployment/provision-explorer-config.sh @@ -39,7 +39,7 @@ eth1GethEndpoint: 'http://127.0.0.1:$EL_PORT' redisCacheEndpoint: '127.0.0.1:$REDIS_PORT' tieredCacheProvider: 'redis' frontend: - siteDomain: "local-testnet.beaconcha.in" + siteDomain: "localhost:8080" siteName: 'Open Source Ethereum (ETH) Testnet Explorer' # Name of the site, displayed in the title tag siteSubtitle: "Showing a local testnet." server: diff --git a/rpc/erigon.go b/rpc/erigon.go index 32a19fd742..84d6577a6d 100644 --- a/rpc/erigon.go +++ b/rpc/erigon.go @@ -235,7 +235,7 @@ func (client *ErigonClient) GetBlock(number int64) (*types.Eth1Block, *types.Get logrus.Fatalf("unknown trace type %v in tx %v", trace.Type, trace.TransactionPosition) } - logger.Infof("appending trace %v to tx %x from %v to %v value %v", trace.TransactionPosition, c.Transactions[trace.TransactionPosition].Hash, trace.From, trace.To, trace.Value) + logger.Tracef("appending trace %v to tx %x from %v to %v value %v", trace.TransactionPosition, c.Transactions[trace.TransactionPosition].Hash, trace.From, trace.To, trace.Value) c.Transactions[trace.TransactionPosition].Itx = append(c.Transactions[trace.TransactionPosition].Itx, tracePb) } From 1f382317fc0276975afe574e0dec198d81acafb7 Mon Sep 17 00:00:00 2001 From: peter <1674920+peterbitfly@users.noreply.github.com> Date: Thu, 31 Aug 2023 17:34:44 +0200 Subject: [PATCH 051/106] (BIDS-2379) address cr issues --- cmd/bundle/main.go | 9 +- cmd/eth1indexer/main.go | 13 +- cmd/signatures/main.go | 4 +- db/bigtable.go | 234 +++++++++++++++-------------- db/bigtable_eth1.go | 12 +- db/node_jobs.go | 6 +- erc20/erc20.go | 4 +- exporter/appsubscription_oracle.go | 4 +- exporter/rocketpool.go | 4 +- handlers/api.go | 6 +- handlers/stripe.go | 3 +- handlers/user.go | 11 +- rpc/lighthouse.go | 4 +- services/notifications.go | 2 +- utils/oauthutils.go | 4 +- utils/utils.go | 10 +- 16 files changed, 167 insertions(+), 163 deletions(-) diff --git a/cmd/bundle/main.go b/cmd/bundle/main.go index 87fd5bdce3..0c5e76ad25 100644 --- a/cmd/bundle/main.go +++ b/cmd/bundle/main.go @@ -4,7 +4,6 @@ import ( "crypto/md5" "eth2-exporter/utils" "fmt" - "io/ioutil" "log" "os" "path" @@ -72,7 +71,7 @@ func bundle(staticDir string) (map[string]string, error) { } for _, match := range matches { - code, err := ioutil.ReadFile(match) + code, err := os.ReadFile(match) if err != nil { return nameMapping, fmt.Errorf("error reading file %v", err) } @@ -97,7 +96,7 @@ func bundle(staticDir string) (map[string]string, error) { newPath := strings.ReplaceAll(matchHash, "static/", "") nameMapping[path] = newPath - err = ioutil.WriteFile(matchHash, code, 0755) + err = os.WriteFile(matchHash, code, 0755) if err != nil { return nameMapping, fmt.Errorf("error failed to write file %v", err) } @@ -116,7 +115,7 @@ func replaceFilesNames(files map[string]string) error { return err } for _, match := range matches { - html, err := ioutil.ReadFile(match) + html, err := os.ReadFile(match) if err != nil { return err } @@ -125,7 +124,7 @@ func replaceFilesNames(files map[string]string) error { // logrus.Info("replacing: ", oldPath, " with: ", newPath) h = strings.ReplaceAll(h, oldPath, newPath) } - err = ioutil.WriteFile(match, []byte(h), 0755) + err = os.WriteFile(match, []byte(h), 0755) if err != nil { return err } diff --git a/cmd/eth1indexer/main.go b/cmd/eth1indexer/main.go index 01197d4b79..6399313ded 100644 --- a/cmd/eth1indexer/main.go +++ b/cmd/eth1indexer/main.go @@ -14,9 +14,10 @@ import ( "eth2-exporter/version" "flag" "fmt" - "io/ioutil" + "io" "math/big" "net/http" + "os" "strconv" "strings" "sync/atomic" @@ -391,7 +392,7 @@ func main() { func UpdateTokenPrices(bt *db.Bigtable, client *rpc.ErigonClient, tokenListPath string) error { - tokenListContent, err := ioutil.ReadFile(tokenListPath) + tokenListContent, err := os.ReadFile(tokenListPath) if err != nil { return err } @@ -431,7 +432,7 @@ func UpdateTokenPrices(bt *db.Bigtable, client *rpc.ErigonClient, tokenListPath return fmt.Errorf("error querying defillama api: %v", resp.Status) } - body, err := ioutil.ReadAll(resp.Body) + body, err := io.ReadAll(resp.Body) if err != nil { return err } @@ -804,7 +805,7 @@ func ImportMainnetERC20TokenMetadataFromTokenDirectory(bt *db.Bigtable) { utils.LogFatal(err, "getting client error", 0) } - body, err := ioutil.ReadAll(resp.Body) + body, err := io.ReadAll(resp.Body) if err != nil { utils.LogFatal(err, "reading body for ERC20 tokens error", 0) @@ -857,7 +858,7 @@ func ImportMainnetERC20TokenMetadataFromTokenDirectory(bt *db.Bigtable) { resp, err := client.Get(token.LogoURI) if err == nil && resp.StatusCode == 200 { - body, err := ioutil.ReadAll(resp.Body) + body, err := io.ReadAll(resp.Body) if err != nil { utils.LogFatal(err, "reading body for ERC20 token logo URI error", 0) @@ -887,7 +888,7 @@ func ImportNameLabels(bt *db.Bigtable) { res := make(map[string]*NameEntry) - data, err := ioutil.ReadFile("") + data, err := os.ReadFile("") if err != nil { utils.LogFatal(err, "reading file error", 0) diff --git a/cmd/signatures/main.go b/cmd/signatures/main.go index b3677948a9..49d3fc6210 100644 --- a/cmd/signatures/main.go +++ b/cmd/signatures/main.go @@ -10,7 +10,7 @@ import ( "eth2-exporter/version" "flag" "fmt" - "io/ioutil" + "io" "net/http" "time" @@ -202,7 +202,7 @@ func GetNextSignatures(bt *db.Bigtable, page string, status types.SignatureImpor return nil, nil, fmt.Errorf("error querying signatures api: %v", resp.Status) } - body, err := ioutil.ReadAll(resp.Body) + body, err := io.ReadAll(resp.Body) if err != nil { return nil, nil, err } diff --git a/db/bigtable.go b/db/bigtable.go index 252cd29390..68af68d0cb 100644 --- a/db/bigtable.go +++ b/db/bigtable.go @@ -37,8 +37,10 @@ const ( SUM_COLUMN = "sum" - max_block_number = 1000000000 - max_epoch = 1000000000 + MAX_BLOCK_NUMBER = 1000000000 - 1 + MAX_EPOCH = 1000000000 - 1 + + MAX_BATCH_MUTATIONS = 100000 ) type Bigtable struct { @@ -132,7 +134,7 @@ func (bigtable *Bigtable) SaveMachineMetric(process string, userID uint64, machi ctx, cancel := context.WithTimeout(context.Background(), time.Second*30) defer cancel() - rowKeyData := fmt.Sprintf("u:%s:p:%s:m:%v", reversePaddedUserID(userID), process, machine) + rowKeyData := fmt.Sprintf("u:%s:p:%s:m:%v", bigtable.reversePaddedUserID(userID), process, machine) ts := gcp_bigtable.Now() rateLimitKey := fmt.Sprintf("%s:%d", rowKeyData, ts.Time().Minute()) @@ -146,7 +148,7 @@ func (bigtable *Bigtable) SaveMachineMetric(process string, userID uint64, machi // for limiting machines per user, add the machine field to a redis set // bucket period is 15mins - machineLimitKey := fmt.Sprintf("%s:%d", reversePaddedUserID(userID), ts.Time().Minute()%15) + machineLimitKey := fmt.Sprintf("%s:%d", bigtable.reversePaddedUserID(userID), ts.Time().Minute()%15) pipe := bigtable.redisCache.Pipeline() pipe.SAdd(ctx, machineLimitKey, machine) pipe.Expire(ctx, machineLimitKey, time.Minute*15) @@ -174,7 +176,7 @@ func (bigtable Bigtable) getMachineMetricNamesMap(userID uint64, searchDepth int ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(time.Second*30)) defer cancel() - rangePrefix := fmt.Sprintf("u:%s:p:", reversePaddedUserID(userID)) + rangePrefix := fmt.Sprintf("u:%s:p:", bigtable.reversePaddedUserID(userID)) filter := gcp_bigtable.ChainFilters( gcp_bigtable.FamilyFilter(MACHINE_METRICS_COLUMN_FAMILY), @@ -219,7 +221,7 @@ func (bigtable Bigtable) GetMachineMetricsMachineCount(userID uint64) (uint64, e ctx, cancel := context.WithTimeout(context.Background(), time.Second*10) defer cancel() - machineLimitKey := fmt.Sprintf("%s:%d", reversePaddedUserID(userID), time.Now().Minute()%15) + machineLimitKey := fmt.Sprintf("%s:%d", bigtable.reversePaddedUserID(userID), time.Now().Minute()%15) card, err := bigtable.redisCache.SCard(ctx, machineLimitKey).Result() if err != nil { @@ -274,7 +276,7 @@ func getMachineMetrics[T types.MachineMetricSystem | types.MachineMetricNode | t ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(time.Second*30)) defer cancel() - rangePrefix := fmt.Sprintf("u:%s:p:%s:m:", reversePaddedUserID(userID), process) + rangePrefix := fmt.Sprintf("u:%s:p:%s:m:", bigtable.reversePaddedUserID(userID), process) res := make([]*T, 0) if offset <= 0 { offset = 1 @@ -314,8 +316,8 @@ func getMachineMetrics[T types.MachineMetricSystem | types.MachineMetricNode | t return res, nil } -func GetMachineRowKey(userID uint64, process string, machine string) string { - return fmt.Sprintf("u:%s:p:%s:m:%s", reversePaddedUserID(userID), process, machine) +func (bigtable Bigtable) GetMachineRowKey(userID uint64, process string, machine string) string { + return fmt.Sprintf("u:%s:p:%s:m:%s", bigtable.reversePaddedUserID(userID), process, machine) } // Returns a map[userID]map[machineName]machineData @@ -415,10 +417,11 @@ func (bigtable *Bigtable) SaveValidatorBalances(epoch uint64, validators []*type // start := time.Now() ts := gcp_bigtable.Timestamp(0) - muts := make([]*gcp_bigtable.Mutation, 0, 100000) - keys := make([]string, 0, 100000) + muts := make([]*gcp_bigtable.Mutation, 0, MAX_BATCH_MUTATIONS) + keys := make([]string, 0, MAX_BATCH_MUTATIONS) - for i, validator := range validators { + epochKey := bigtable.reversedPaddedEpoch(epoch) + for _, validator := range validators { balanceEncoded := make([]byte, 8) binary.LittleEndian.PutUint64(balanceEncoded, validator.Balance) @@ -428,12 +431,12 @@ func (bigtable *Bigtable) SaveValidatorBalances(epoch uint64, validators []*type combined := append(balanceEncoded, effectiveBalanceEncoded...) mut := &gcp_bigtable.Mutation{} mut.Set(VALIDATOR_BALANCES_FAMILY, "b", ts, combined) - key := fmt.Sprintf("%s:%s:%s", bigtable.chainId, bigtable.validatorIndexToKey(validator.Index), reversedPaddedEpoch(epoch)) + key := fmt.Sprintf("%s:%s:%s", bigtable.chainId, bigtable.validatorIndexToKey(validator.Index), epochKey) muts = append(muts, mut) keys = append(keys, key) - if i%100000 == 0 { + if len(muts) == MAX_BATCH_MUTATIONS { errs, err := bigtable.tableValidatorBalances.ApplyBulk(ctx, keys, muts) if err != nil { @@ -443,8 +446,8 @@ func (bigtable *Bigtable) SaveValidatorBalances(epoch uint64, validators []*type for _, err := range errs { return err } - muts = make([]*gcp_bigtable.Mutation, 0, 100000) - keys = make([]string, 0, 100000) + muts = make([]*gcp_bigtable.Mutation, 0, MAX_BATCH_MUTATIONS) + keys = make([]string, 0, MAX_BATCH_MUTATIONS) } } @@ -487,21 +490,21 @@ func (bigtable *Bigtable) SaveAttestationAssignments(epoch uint64, assignments m validatorsPerSlot[attesterslot] = append(validatorsPerSlot[attesterslot], validator) } - muts := make([]*gcp_bigtable.Mutation, 0, 100000) - keys := make([]string, 0, 100000) + muts := make([]*gcp_bigtable.Mutation, 0, MAX_BATCH_MUTATIONS) + keys := make([]string, 0, MAX_BATCH_MUTATIONS) for slot, validators := range validatorsPerSlot { for _, validator := range validators { mut := gcp_bigtable.NewMutation() - key := fmt.Sprintf("%s:%s:%s", bigtable.chainId, bigtable.validatorIndexToKey(validator), reversedPaddedEpoch(epoch)) + key := fmt.Sprintf("%s:%s:%s", bigtable.chainId, bigtable.validatorIndexToKey(validator), bigtable.reversedPaddedEpoch(epoch)) mut.Set(ATTESTATIONS_FAMILY, fmt.Sprintf("%d", slot), ts, []byte{}) muts = append(muts, mut) keys = append(keys, key) } - if len(muts)%100000 == 0 { + if len(muts) == MAX_BATCH_MUTATIONS { errs, err := bigtable.tableValidatorAttestations.ApplyBulk(ctx, keys, muts) if err != nil { @@ -511,8 +514,8 @@ func (bigtable *Bigtable) SaveAttestationAssignments(epoch uint64, assignments m for _, err := range errs { return err } - muts = make([]*gcp_bigtable.Mutation, 0, 100000) - keys = make([]string, 0, 100000) + muts = make([]*gcp_bigtable.Mutation, 0, MAX_BATCH_MUTATIONS) + keys = make([]string, 0, MAX_BATCH_MUTATIONS) } } @@ -539,14 +542,14 @@ func (bigtable *Bigtable) SaveProposalAssignments(epoch uint64, assignments map[ start := time.Now() ts := gcp_bigtable.Timestamp(0) - muts := make([]*gcp_bigtable.Mutation, 0, 100000) - keys := make([]string, 0, 100000) + muts := make([]*gcp_bigtable.Mutation, 0, len(assignments)) + keys := make([]string, 0, len(assignments)) for slot, validator := range assignments { mut := gcp_bigtable.NewMutation() mut.Set(PROPOSALS_FAMILY, "p", ts, []byte{}) - key := fmt.Sprintf("%s:%s:%s:%s", bigtable.chainId, bigtable.validatorIndexToKey(validator), reversedPaddedEpoch(epoch), reversedPaddedSlot(slot)) + key := fmt.Sprintf("%s:%s:%s:%s", bigtable.chainId, bigtable.validatorIndexToKey(validator), bigtable.reversedPaddedEpoch(epoch), bigtable.reversedPaddedSlot(slot)) muts = append(muts, mut) keys = append(keys, key) @@ -576,20 +579,20 @@ func (bigtable *Bigtable) SaveSyncCommitteesAssignments(startSlot, endSlot uint6 start := time.Now() ts := gcp_bigtable.Timestamp(0) - var muts []*gcp_bigtable.Mutation - var keys []string + muts := make([]*gcp_bigtable.Mutation, 0, MAX_BATCH_MUTATIONS) + keys := make([]string, 0, MAX_BATCH_MUTATIONS) for i := startSlot; i <= endSlot; i++ { for _, validator := range validators { mut := gcp_bigtable.NewMutation() mut.Set(SYNC_COMMITTEES_FAMILY, "s", ts, []byte{}) - key := fmt.Sprintf("%s:%s:%s:%s", bigtable.chainId, bigtable.validatorIndexToKey(validator), reversedPaddedEpoch(i/utils.Config.Chain.Config.SlotsPerEpoch), reversedPaddedSlot(i)) + key := fmt.Sprintf("%s:%s:%s:%s", bigtable.chainId, bigtable.validatorIndexToKey(validator), bigtable.reversedPaddedEpoch(utils.EpochOfSlot(i)), bigtable.reversedPaddedSlot(i)) muts = append(muts, mut) keys = append(keys, key) - if len(muts)%100000 == 0 { + if len(muts) == MAX_BATCH_MUTATIONS { logger.Infof("saving %v mutations for sync duties", len(muts)) errs, err := bigtable.tableValidatorSyncCommittees.ApplyBulk(ctx, keys, muts) @@ -603,8 +606,8 @@ func (bigtable *Bigtable) SaveSyncCommitteesAssignments(startSlot, endSlot uint6 } } - muts = make([]*gcp_bigtable.Mutation, 0, 100000) - keys = make([]string, 0, 100000) + muts = make([]*gcp_bigtable.Mutation, 0, MAX_BATCH_MUTATIONS) + keys = make([]string, 0, MAX_BATCH_MUTATIONS) } } @@ -681,20 +684,20 @@ func (bigtable *Bigtable) SaveAttestations(blocks map[uint64]map[string]*types.B } } - mutsInclusionSlot := make([]*gcp_bigtable.Mutation, 0, 100000) - keysInclusionSlot := make([]string, 0, 100000) + mutsInclusionSlot := make([]*gcp_bigtable.Mutation, 0, MAX_BATCH_MUTATIONS) + keysInclusionSlot := make([]string, 0, MAX_BATCH_MUTATIONS) for attestedSlot, inclusions := range attestationsBySlot { mutLastAttestationSlot := gcp_bigtable.NewMutation() mutLastAttestationSlotSet := false - epoch := attestedSlot / utils.Config.Chain.Config.SlotsPerEpoch + epoch := utils.EpochOfSlot(attestedSlot) bigtable.lastAttestationCacheMux.Lock() for validator, inclusionSlot := range inclusions { mutInclusionSlot := gcp_bigtable.NewMutation() - mutInclusionSlot.Set(ATTESTATIONS_FAMILY, fmt.Sprintf("%d", attestedSlot), gcp_bigtable.Timestamp((max_block_number-inclusionSlot)*1000), []byte{}) - key := fmt.Sprintf("%s:%s:%s", bigtable.chainId, bigtable.validatorIndexToKey(validator), reversedPaddedEpoch(epoch)) + mutInclusionSlot.Set(ATTESTATIONS_FAMILY, fmt.Sprintf("%d", attestedSlot), gcp_bigtable.Timestamp((MAX_BLOCK_NUMBER-inclusionSlot)*1000), []byte{}) + key := fmt.Sprintf("%s:%s:%s", bigtable.chainId, bigtable.validatorIndexToKey(validator), bigtable.reversedPaddedEpoch(epoch)) mutsInclusionSlot = append(mutsInclusionSlot, mutInclusionSlot) keysInclusionSlot = append(keysInclusionSlot, key) @@ -709,7 +712,7 @@ func (bigtable *Bigtable) SaveAttestations(blocks map[uint64]map[string]*types.B attstart := time.Now() - if len(mutsInclusionSlot)%100000 == 0 { + if len(mutsInclusionSlot) == MAX_BATCH_MUTATIONS { errs, err := bigtable.tableValidatorAttestations.ApplyBulk(ctx, keysInclusionSlot, mutsInclusionSlot) if err != nil { return err @@ -718,8 +721,8 @@ func (bigtable *Bigtable) SaveAttestations(blocks map[uint64]map[string]*types.B return err } logger.Infof("applied attestation mutations in %v", time.Since(attstart)) - mutsInclusionSlot = make([]*gcp_bigtable.Mutation, 0, 100000) - keysInclusionSlot = make([]string, 0, 100000) + mutsInclusionSlot = make([]*gcp_bigtable.Mutation, 0, MAX_BATCH_MUTATIONS) + keysInclusionSlot = make([]string, 0, MAX_BATCH_MUTATIONS) } if mutLastAttestationSlotSet { err := bigtable.tableValidators.Apply(ctx, fmt.Sprintf("%s:lastAttestationSlot", bigtable.chainId), mutLastAttestationSlot) @@ -775,8 +778,8 @@ func (bigtable *Bigtable) SaveProposals(blocks map[uint64]map[string]*types.Bloc return slots[i] < slots[j] }) - muts := make([]*gcp_bigtable.Mutation, 0, 100000) - keys := make([]string, 0, 100000) + muts := make([]*gcp_bigtable.Mutation, 0, MAX_BATCH_MUTATIONS) + keys := make([]string, 0, MAX_BATCH_MUTATIONS) for _, slot := range slots { for _, b := range blocks[slot] { @@ -785,8 +788,8 @@ func (bigtable *Bigtable) SaveProposals(blocks map[uint64]map[string]*types.Bloc continue } mut := gcp_bigtable.NewMutation() - mut.Set(PROPOSALS_FAMILY, "b", gcp_bigtable.Timestamp((max_block_number-b.Slot)*1000), []byte{}) - key := fmt.Sprintf("%s:%s:%s:%s", bigtable.chainId, bigtable.validatorIndexToKey(b.Proposer), reversedPaddedEpoch(b.Slot/utils.Config.Chain.Config.SlotsPerEpoch), reversedPaddedSlot(b.Slot)) + mut.Set(PROPOSALS_FAMILY, "b", gcp_bigtable.Timestamp((MAX_BLOCK_NUMBER-b.Slot)*1000), []byte{}) + key := fmt.Sprintf("%s:%s:%s:%s", bigtable.chainId, bigtable.validatorIndexToKey(b.Proposer), bigtable.reversedPaddedEpoch(utils.EpochOfSlot(b.Slot)), bigtable.reversedPaddedSlot(b.Slot)) muts = append(muts, mut) keys = append(keys, key) @@ -847,18 +850,18 @@ func (bigtable *Bigtable) SaveSyncComitteeDuties(blocks map[uint64]map[string]*t return nil } - muts := make([]*gcp_bigtable.Mutation, 0, 100000) - keys := make([]string, 0, 100000) + muts := make([]*gcp_bigtable.Mutation, 0, MAX_BATCH_MUTATIONS) + keys := make([]string, 0, MAX_BATCH_MUTATIONS) for slot, validators := range dutiesBySlot { for validator, participated := range validators { mut := gcp_bigtable.NewMutation() if participated { - mut.Set(SYNC_COMMITTEES_FAMILY, "s", gcp_bigtable.Timestamp((max_block_number-slot)*1000), []byte{}) + mut.Set(SYNC_COMMITTEES_FAMILY, "s", gcp_bigtable.Timestamp((MAX_BLOCK_NUMBER-slot)*1000), []byte{}) } else { mut.Set(SYNC_COMMITTEES_FAMILY, "s", gcp_bigtable.Timestamp(0), []byte{}) } - key := fmt.Sprintf("%s:%s:%s:%s", bigtable.chainId, bigtable.validatorIndexToKey(validator), reversedPaddedEpoch(slot/utils.Config.Chain.Config.SlotsPerEpoch), reversedPaddedSlot(slot)) + key := fmt.Sprintf("%s:%s:%s:%s", bigtable.chainId, bigtable.validatorIndexToKey(validator), bigtable.reversedPaddedEpoch(utils.EpochOfSlot(slot)), bigtable.reversedPaddedSlot(slot)) muts = append(muts, mut) keys = append(keys, key) } @@ -953,7 +956,7 @@ func (bigtable *Bigtable) GetValidatorBalanceHistory(validators []uint64, startE resMux.Lock() res[validator] = append(res[validator], &types.ValidatorBalance{ - Epoch: max_epoch - epoch, + Epoch: MAX_EPOCH - epoch, Balance: balance, EffectiveBalance: effectiveBalance, Index: validator, @@ -1042,10 +1045,10 @@ func (bigtable *Bigtable) GetValidatorAttestationHistory(validators []uint64, st logger.Errorf("error parsing slot from row key %v: %v", r.Key(), err) return false } - inclusionSlot := max_block_number - uint64(ri.Timestamp)/1000 + inclusionSlot := MAX_BLOCK_NUMBER - uint64(ri.Timestamp)/1000 status := uint64(1) - if inclusionSlot == max_block_number { + if inclusionSlot == MAX_BLOCK_NUMBER { inclusionSlot = 0 status = 0 } else if orphanedSlotsMap[inclusionSlot] { @@ -1067,7 +1070,7 @@ func (bigtable *Bigtable) GetValidatorAttestationHistory(validators []uint64, st } else { res[validator] = append(res[validator], &types.ValidatorAttestation{ Index: validator, - Epoch: attesterSlot / utils.Config.Chain.Config.SlotsPerEpoch, + Epoch: utils.EpochOfSlot(attesterSlot), AttesterSlot: attesterSlot, CommitteeIndex: 0, Status: status, @@ -1212,10 +1215,10 @@ func (bigtable *Bigtable) GetValidatorMissedAttestationHistory(validators []uint return false } - inclusionSlot := max_block_number - uint64(ri.Timestamp)/1000 + inclusionSlot := MAX_BLOCK_NUMBER - uint64(ri.Timestamp)/1000 status := uint64(1) - if inclusionSlot == max_block_number { + if inclusionSlot == MAX_BLOCK_NUMBER { status = 0 } @@ -1302,11 +1305,11 @@ func (bigtable *Bigtable) GetValidatorSyncDutiesHistory(validators []uint64, sta logger.Errorf("error parsing slot from row key %v: %v", r.Key(), err) return false } - slot = max_block_number - slot - inclusionSlot := max_block_number - uint64(ri.Timestamp)/1000 + slot = MAX_BLOCK_NUMBER - slot + inclusionSlot := MAX_BLOCK_NUMBER - uint64(ri.Timestamp)/1000 status := uint64(1) // 1: participated - if inclusionSlot == max_block_number { + if inclusionSlot == MAX_BLOCK_NUMBER { inclusionSlot = 0 status = 0 // 0: missed } @@ -1580,11 +1583,11 @@ func (bigtable *Bigtable) GetValidatorProposalHistory(validators []uint64, start logger.Errorf("error parsing slot from row key %v: %v", r.Key(), err) return false } - proposalSlot = max_block_number - proposalSlot - inclusionSlot := max_block_number - uint64(r[PROPOSALS_FAMILY][0].Timestamp)/1000 + proposalSlot = MAX_BLOCK_NUMBER - proposalSlot + inclusionSlot := MAX_BLOCK_NUMBER - uint64(r[PROPOSALS_FAMILY][0].Timestamp)/1000 status := uint64(1) - if inclusionSlot == max_block_number { + if inclusionSlot == MAX_BLOCK_NUMBER { inclusionSlot = 0 status = 2 } @@ -1636,8 +1639,8 @@ func (bigtable *Bigtable) SaveValidatorIncomeDetails(epoch uint64, rewards map[u total := &itypes.ValidatorEpochIncome{} - muts := make([]*gcp_bigtable.Mutation, 0, 100000) - keys := make([]string, 0, 100000) + muts := make([]*gcp_bigtable.Mutation, 0, MAX_BATCH_MUTATIONS) + keys := make([]string, 0, MAX_BATCH_MUTATIONS) mutsCount := 0 for i, rewardDetails := range rewards { @@ -1650,13 +1653,13 @@ func (bigtable *Bigtable) SaveValidatorIncomeDetails(epoch uint64, rewards map[u mut := &gcp_bigtable.Mutation{} mut.Set(INCOME_DETAILS_COLUMN_FAMILY, "i", ts, data) - key := fmt.Sprintf("%s:%s:%s", bigtable.chainId, bigtable.validatorIndexToKey(i), reversedPaddedEpoch(epoch)) + key := fmt.Sprintf("%s:%s:%s", bigtable.chainId, bigtable.validatorIndexToKey(i), bigtable.reversedPaddedEpoch(epoch)) muts = append(muts, mut) keys = append(keys, key) mutsCount++ - if mutsCount%100000 == 0 { + if len(muts) == MAX_BATCH_MUTATIONS { errs, err := bigtable.tableValidatorIncomeDetails.ApplyBulk(ctx, keys, muts) if err != nil { @@ -1665,8 +1668,8 @@ func (bigtable *Bigtable) SaveValidatorIncomeDetails(epoch uint64, rewards map[u for _, err := range errs { return err } - muts = make([]*gcp_bigtable.Mutation, 0, 100000) - keys = make([]string, 0, 100000) + muts = make([]*gcp_bigtable.Mutation, 0, MAX_BATCH_MUTATIONS) + keys = make([]string, 0, MAX_BATCH_MUTATIONS) } total.AttestationHeadReward += rewardDetails.AttestationHeadReward @@ -1685,6 +1688,17 @@ func (bigtable *Bigtable) SaveValidatorIncomeDetails(epoch uint64, rewards map[u total.TxFeeRewardWei = utils.AddBigInts(total.TxFeeRewardWei, rewardDetails.TxFeeRewardWei) } + if len(muts) > 0 { + errs, err := bigtable.tableValidatorIncomeDetails.ApplyBulk(ctx, keys, muts) + + if err != nil { + return err + } + for _, err := range errs { + return err + } + } + sum, err := proto.Marshal(total) if err != nil { return err @@ -1693,7 +1707,7 @@ func (bigtable *Bigtable) SaveValidatorIncomeDetails(epoch uint64, rewards map[u mut := &gcp_bigtable.Mutation{} mut.Set(STATS_COLUMN_FAMILY, SUM_COLUMN, ts, sum) - err = bigtable.tableValidatorIncomeDetails.Apply(ctx, fmt.Sprintf("%s:%s:%s", bigtable.chainId, SUM_COLUMN, reversedPaddedEpoch(epoch)), mut) + err = bigtable.tableValidatorIncomeDetails.Apply(ctx, fmt.Sprintf("%s:%s:%s", bigtable.chainId, SUM_COLUMN, bigtable.reversedPaddedEpoch(epoch)), mut) if err != nil { return err } @@ -1726,7 +1740,7 @@ func (bigtable *Bigtable) MigrateEpochSchemaV1ToV2(epoch uint64) error { filter := gcp_bigtable.LatestNFilter(1) ctx := context.Background() - prefixEpochRange := gcp_bigtable.PrefixRange(fmt.Sprintf("%s:e:b:%s", bigtable.chainId, reversedPaddedEpoch(epoch))) + prefixEpochRange := gcp_bigtable.PrefixRange(fmt.Sprintf("%s:e:b:%s", bigtable.chainId, bigtable.reversedPaddedEpoch(epoch))) err := bigtable.tableBeaconchain.ReadRows(ctx, prefixEpochRange, func(r gcp_bigtable.Row) bool { // logger.Infof("processing row %v", r.Key()) @@ -1739,7 +1753,7 @@ func (bigtable *Bigtable) MigrateEpochSchemaV1ToV2(epoch uint64) error { return false } - rowKeyEpoch = max_epoch - rowKeyEpoch + rowKeyEpoch = MAX_EPOCH - rowKeyEpoch if epoch != rowKeyEpoch { logger.Errorf("retrieved different epoch than requested, requested: %d, retrieved: %d", epoch, rowKeyEpoch) @@ -1804,7 +1818,7 @@ func (bigtable *Bigtable) MigrateEpochSchemaV1ToV2(epoch uint64) error { // logger.Infof("retrieved epoch data for %d validators in %v", len(epochData), time.Since(start)) // start = time.Now() - prefixEpochSlotRange := gcp_bigtable.PrefixRange(fmt.Sprintf("%s:e:%s:s:", bigtable.chainId, reversedPaddedEpoch(epoch))) + prefixEpochSlotRange := gcp_bigtable.PrefixRange(fmt.Sprintf("%s:e:%s:s:", bigtable.chainId, bigtable.reversedPaddedEpoch(epoch))) err = bigtable.tableBeaconchain.ReadRows(ctx, prefixEpochSlotRange, func(r gcp_bigtable.Row) bool { // logger.Infof("processing row %v", r.Key()) @@ -1817,7 +1831,7 @@ func (bigtable *Bigtable) MigrateEpochSchemaV1ToV2(epoch uint64) error { return false } - rowKeyEpoch = max_epoch - rowKeyEpoch + rowKeyEpoch = MAX_EPOCH - rowKeyEpoch if epoch != rowKeyEpoch { logger.Errorf("retrieved different epoch than requested, requested: %d, retrieved: %d", epoch, rowKeyEpoch) @@ -1828,7 +1842,7 @@ func (bigtable *Bigtable) MigrateEpochSchemaV1ToV2(epoch uint64) error { logger.Errorf("error parsing slot from row key %v: %v", r.Key(), err) return false } - slot = max_block_number - slot + slot = MAX_BLOCK_NUMBER - slot // logger.Infof("epoch is %d, slot is %d", rowKeyEpoch, slot) @@ -1845,14 +1859,16 @@ func (bigtable *Bigtable) MigrateEpochSchemaV1ToV2(epoch uint64) error { inclusionSlot := uint64(0) if ri.Timestamp > 0 { - inclusionSlot = max_block_number - uint64(ri.Timestamp)/1000 + inclusionSlot = MAX_BLOCK_NUMBER - uint64(ri.Timestamp)/1000 } // logger.Infof("retrieved field %s from column family %s for validator %d", ri.Column, columnFamily, validator) if epochData[validator] == nil { epochData[validator] = &validatorEpochData{ - ValidatorIndex: validator, + ValidatorIndex: validator, + Proposals: make(map[uint64]uint64), + SyncParticipation: make(map[uint64]uint64), } } @@ -1902,18 +1918,18 @@ func (bigtable *Bigtable) MigrateEpochSchemaV1ToV2(epoch uint64) error { // start = time.Now() i := 0 - mutsInclusionSlot := make([]*gcp_bigtable.Mutation, 0, 100000) - keysInclusionSlot := make([]string, 0, 100000) + mutsInclusionSlot := make([]*gcp_bigtable.Mutation, 0, MAX_BATCH_MUTATIONS) + keysInclusionSlot := make([]string, 0, MAX_BATCH_MUTATIONS) for _, validator := range epochData { mutInclusionSlot := gcp_bigtable.NewMutation() - mutInclusionSlot.Set(ATTESTATIONS_FAMILY, fmt.Sprintf("%d", validator.AttestationTargetSlot), gcp_bigtable.Timestamp((max_block_number-validator.AttestationInclusionSlot)*1000), []byte{}) - key := fmt.Sprintf("%s:%s:%s", bigtable.chainId, bigtable.validatorIndexToKey(validator.ValidatorIndex), reversedPaddedEpoch(epoch)) + mutInclusionSlot.Set(ATTESTATIONS_FAMILY, fmt.Sprintf("%d", validator.AttestationTargetSlot), gcp_bigtable.Timestamp((MAX_BLOCK_NUMBER-validator.AttestationInclusionSlot)*1000), []byte{}) + key := fmt.Sprintf("%s:%s:%s", bigtable.chainId, bigtable.validatorIndexToKey(validator.ValidatorIndex), bigtable.reversedPaddedEpoch(epoch)) mutsInclusionSlot = append(mutsInclusionSlot, mutInclusionSlot) keysInclusionSlot = append(keysInclusionSlot, key) - if i%100000 == 0 { + if len(mutsInclusionSlot) == MAX_BATCH_MUTATIONS { errs, err := bigtable.tableValidatorAttestations.ApplyBulk(ctx, keysInclusionSlot, mutsInclusionSlot) if err != nil { return err @@ -1921,8 +1937,8 @@ func (bigtable *Bigtable) MigrateEpochSchemaV1ToV2(epoch uint64) error { for _, err := range errs { return err } - mutsInclusionSlot = make([]*gcp_bigtable.Mutation, 0, 100000) - keysInclusionSlot = make([]string, 0, 100000) + mutsInclusionSlot = make([]*gcp_bigtable.Mutation, 0, MAX_BATCH_MUTATIONS) + keysInclusionSlot = make([]string, 0, MAX_BATCH_MUTATIONS) } i++ } @@ -1941,8 +1957,8 @@ func (bigtable *Bigtable) MigrateEpochSchemaV1ToV2(epoch uint64) error { // logger.Infof("migrated attestation data in %v", time.Since(start)) // start = time.Now() - mutsProposals := make([]*gcp_bigtable.Mutation, 0, 100000) - keysProposals := make([]string, 0, 100000) + mutsProposals := make([]*gcp_bigtable.Mutation, 0, MAX_BATCH_MUTATIONS) + keysProposals := make([]string, 0, MAX_BATCH_MUTATIONS) for _, validator := range epochData { if len(validator.Proposals) == 0 { @@ -1950,8 +1966,8 @@ func (bigtable *Bigtable) MigrateEpochSchemaV1ToV2(epoch uint64) error { } for slot, inclusionSlot := range validator.Proposals { mut := gcp_bigtable.NewMutation() - mut.Set(PROPOSALS_FAMILY, "b", gcp_bigtable.Timestamp((max_block_number-inclusionSlot)*1000), []byte{}) - key := fmt.Sprintf("%s:%s:%s:%s", bigtable.chainId, bigtable.validatorIndexToKey(validator.ValidatorIndex), reversedPaddedEpoch(epoch), reversedPaddedSlot(slot)) + mut.Set(PROPOSALS_FAMILY, "b", gcp_bigtable.Timestamp((MAX_BLOCK_NUMBER-inclusionSlot)*1000), []byte{}) + key := fmt.Sprintf("%s:%s:%s:%s", bigtable.chainId, bigtable.validatorIndexToKey(validator.ValidatorIndex), bigtable.reversedPaddedEpoch(epoch), bigtable.reversedPaddedSlot(slot)) mutsProposals = append(mutsProposals, mut) keysProposals = append(keysProposals, key) @@ -1969,8 +1985,8 @@ func (bigtable *Bigtable) MigrateEpochSchemaV1ToV2(epoch uint64) error { // logger.Infof("migrated proposal data in %v", time.Since(start)) // start = time.Now() - mutsSync := make([]*gcp_bigtable.Mutation, 0, 100000) - keysSync := make([]string, 0, 100000) + mutsSync := make([]*gcp_bigtable.Mutation, 0, MAX_BATCH_MUTATIONS) + keysSync := make([]string, 0, MAX_BATCH_MUTATIONS) for _, validator := range epochData { @@ -1979,9 +1995,9 @@ func (bigtable *Bigtable) MigrateEpochSchemaV1ToV2(epoch uint64) error { } for slot, inclusionSlot := range validator.SyncParticipation { mut := gcp_bigtable.NewMutation() - mut.Set(SYNC_COMMITTEES_FAMILY, "s", gcp_bigtable.Timestamp((max_block_number-inclusionSlot)*1000), []byte{}) + mut.Set(SYNC_COMMITTEES_FAMILY, "s", gcp_bigtable.Timestamp((MAX_BLOCK_NUMBER-inclusionSlot)*1000), []byte{}) - key := fmt.Sprintf("%s:%s:%s:%s", bigtable.chainId, bigtable.validatorIndexToKey(validator.ValidatorIndex), reversedPaddedEpoch(epoch), reversedPaddedSlot(slot)) + key := fmt.Sprintf("%s:%s:%s:%s", bigtable.chainId, bigtable.validatorIndexToKey(validator.ValidatorIndex), bigtable.reversedPaddedEpoch(epoch), bigtable.reversedPaddedSlot(slot)) mutsSync = append(mutsSync, mut) keysSync = append(keysSync, key) } @@ -2088,7 +2104,7 @@ func (bigtable *Bigtable) GetValidatorIncomeDetailsHistory(validators []uint64, res[validator] = make(map[uint64]*itypes.ValidatorEpochIncome) } - res[validator][max_epoch-epoch] = incomeDetails + res[validator][MAX_EPOCH-epoch] = incomeDetails resMux.Unlock() } return true @@ -2105,7 +2121,7 @@ func (bigtable *Bigtable) GetValidatorIncomeDetailsHistory(validators []uint64, return res, nil } -// GetValidatorIncomeDetailsHistory returns the validator income details +// GetAggregatedValidatorIncomeDetailsHistory returns aggregated validator income details // startEpoch & endEpoch are inclusive func (bigtable *Bigtable) GetAggregatedValidatorIncomeDetailsHistory(validators []uint64, startEpoch uint64, endEpoch uint64) (map[uint64]*itypes.ValidatorEpochIncome, error) { if startEpoch > endEpoch { @@ -2167,6 +2183,7 @@ func (bigtable *Bigtable) GetAggregatedValidatorIncomeDetailsHistory(validators // Deletes all block data from bigtable func (bigtable *Bigtable) DeleteEpoch(epoch uint64) error { + // TOTO: Implement return fmt.Errorf("NOT IMPLEMENTED") } @@ -2179,25 +2196,14 @@ func (bigtable *Bigtable) getValidatorsEpochRanges(validatorIndices []uint64, st ranges := make(gcp_bigtable.RowRangeList, 0, int((endEpoch-startEpoch))*len(validatorIndices)) for _, validatorIndex := range validatorIndices { - if startEpoch == 0 { // special case when the 0 epoch is included - rangeEnd := fmt.Sprintf("%s:%s:%s%s", bigtable.chainId, bigtable.validatorIndexToKey(validatorIndex), reversedPaddedEpoch(0), "\x00") - rangeStart := fmt.Sprintf("%s:%s:%s", bigtable.chainId, bigtable.validatorIndexToKey(validatorIndex), reversedPaddedEpoch(0)) - ranges = append(ranges, gcp_bigtable.NewRange(rangeStart, rangeEnd)) - - // epochs are sorted descending, so start with the largest epoch and end with the smallest - // add \x00 to make the range inclusive - if startEpoch < endEpoch { - rangeEnd = fmt.Sprintf("%s:%s:%s%s", bigtable.chainId, bigtable.validatorIndexToKey(validatorIndex), reversedPaddedEpoch(startEpoch+1), "\x00") - rangeStart = fmt.Sprintf("%s:%s:%s", bigtable.chainId, bigtable.validatorIndexToKey(validatorIndex), reversedPaddedEpoch(endEpoch)) - ranges = append(ranges, gcp_bigtable.NewRange(rangeStart, rangeEnd)) - } - } else { - // epochs are sorted descending, so start with the largest epoch and end with the smallest - // add \x00 to make the range inclusive - rangeEnd := fmt.Sprintf("%s:%s:%s%s", bigtable.chainId, bigtable.validatorIndexToKey(validatorIndex), reversedPaddedEpoch(startEpoch), "\x00") - rangeStart := fmt.Sprintf("%s:%s:%s", bigtable.chainId, bigtable.validatorIndexToKey(validatorIndex), reversedPaddedEpoch(endEpoch)) - ranges = append(ranges, gcp_bigtable.NewRange(rangeStart, rangeEnd)) - } + validatorKey := bigtable.validatorIndexToKey(validatorIndex) + + // epochs are sorted descending, so start with the largest epoch and end with the smallest + // add \x00 to make the range inclusive + rangeEnd := fmt.Sprintf("%s:%s:%s%s", bigtable.chainId, validatorKey, bigtable.reversedPaddedEpoch(startEpoch), "\x00") + rangeStart := fmt.Sprintf("%s:%s:%s", bigtable.chainId, bigtable.validatorIndexToKey(validatorIndex), bigtable.reversedPaddedEpoch(endEpoch)) + ranges = append(ranges, gcp_bigtable.NewRange(rangeStart, rangeEnd)) + } return ranges } @@ -2286,14 +2292,14 @@ func GetCurrentDayClIncome(validator_indices []uint64) (map[uint64]int64, error) return dayIncome, nil } -func reversePaddedUserID(userID uint64) string { +func (bigtable *Bigtable) reversePaddedUserID(userID uint64) string { return fmt.Sprintf("%09d", ^uint64(0)-userID) } -func reversedPaddedEpoch(epoch uint64) string { - return fmt.Sprintf("%09d", max_block_number-epoch) +func (bigtable *Bigtable) reversedPaddedEpoch(epoch uint64) string { + return fmt.Sprintf("%09d", MAX_BLOCK_NUMBER-epoch) } -func reversedPaddedSlot(slot uint64) string { - return fmt.Sprintf("%09d", max_block_number-slot) +func (bigtable *Bigtable) reversedPaddedSlot(slot uint64) string { + return fmt.Sprintf("%09d", MAX_BLOCK_NUMBER-slot) } diff --git a/db/bigtable_eth1.go b/db/bigtable_eth1.go index bc520af229..19a24102d0 100644 --- a/db/bigtable_eth1.go +++ b/db/bigtable_eth1.go @@ -176,7 +176,7 @@ func (bigtable *Bigtable) CheckForGapsInBlocksTable(lookback int) (gapFound bool logger.Errorf("error parsing block number from key %v: %v", r.Key(), err) return false } - c = max_block_number - c + c = MAX_BLOCK_NUMBER - c if c%10000 == 0 { logger.Infof("scanning, currently at block %v", c) @@ -251,7 +251,7 @@ func (bigtable *Bigtable) CheckForGapsInDataTable(lookback int) error { logger.Errorf("error parsing block number from key %v: %v", r.Key(), err) return false } - c = max_block_number - c + c = MAX_BLOCK_NUMBER - c if c%10000 == 0 { logger.Infof("scanning, currently at block %v", c) @@ -316,7 +316,7 @@ func (bigtable *Bigtable) getLastBlockInDataTableFromBigtable() (int, error) { logger.Errorf("error parsing block number from key %v: %v", r.Key(), err) return false } - c = max_block_number - c + c = MAX_BLOCK_NUMBER - c lastBlock = c return c == 0 // required as the block with number 0 will be returned as first block before the most recent one @@ -343,7 +343,7 @@ func (bigtable *Bigtable) getLastBlockInBlocksTableFromBigtable() (int, error) { logger.Errorf("error parsing block number from key %v: %v", r.Key(), err) return false } - c = max_block_number - c + c = MAX_BLOCK_NUMBER - c lastBlock = c return c == 0 // required as the block with number 0 will be returned as first block before the most recent one @@ -382,7 +382,7 @@ func (bigtable *Bigtable) GetMostRecentBlockFromDataTable() (*types.Eth1BlockInd return false } - c = max_block_number - c + c = MAX_BLOCK_NUMBER - c err = proto.Unmarshal(row[DEFAULT_FAMILY][0].Value, &block) if err != nil { @@ -539,7 +539,7 @@ func (bigtable *Bigtable) GetBlocksDescending(start, limit uint64) ([]*types.Eth } func reversedPaddedBlockNumber(blockNumber uint64) string { - return fmt.Sprintf("%09d", max_block_number-blockNumber) + return fmt.Sprintf("%09d", MAX_BLOCK_NUMBER-blockNumber) } func reversePaddedBigtableTimestamp(timestamp *timestamppb.Timestamp) string { diff --git a/db/node_jobs.go b/db/node_jobs.go index ea01189d08..81da90d593 100644 --- a/db/node_jobs.go +++ b/db/node_jobs.go @@ -6,7 +6,7 @@ import ( "eth2-exporter/types" "eth2-exporter/utils" "fmt" - "io/ioutil" + "io" "net/http" "strings" "time" @@ -311,7 +311,7 @@ func SubmitBLSToExecutionChangesNodeJob(job *types.NodeJob) error { } jobStatus := types.SubmittedToNodeNodeJobStatus if resp.StatusCode != 200 { - d, _ := ioutil.ReadAll(resp.Body) + d, _ := io.ReadAll(resp.Body) if len(d) > 1000 { d = d[:1000] } @@ -447,7 +447,7 @@ func SubmitVoluntaryExitNodeJob(job *types.NodeJob) error { } jobStatus := types.SubmittedToNodeNodeJobStatus if resp.StatusCode != 200 { - d, _ := ioutil.ReadAll(resp.Body) + d, _ := io.ReadAll(resp.Body) if len(d) > 1000 { d = d[:1000] } diff --git a/erc20/erc20.go b/erc20/erc20.go index ead654a2bf..c6660327d0 100644 --- a/erc20/erc20.go +++ b/erc20/erc20.go @@ -4,8 +4,8 @@ import ( "encoding/json" "eth2-exporter/utils" "fmt" - "io/ioutil" "math/big" + "os" "strings" "github.com/ethereum/go-ethereum/accounts/abi" @@ -22,7 +22,7 @@ var tokenMap = make(map[string]*ERC20TokenDetail) var logger = logrus.StandardLogger().WithField("module", "erc20") func InitTokenList(path string) { - body, err := ioutil.ReadFile(path) + body, err := os.ReadFile(path) if err != nil { utils.LogFatal(err, "unable to retrieve erc20 token list", 0) } diff --git a/exporter/appsubscription_oracle.go b/exporter/appsubscription_oracle.go index afcfa108f8..bd640bb252 100644 --- a/exporter/appsubscription_oracle.go +++ b/exporter/appsubscription_oracle.go @@ -8,8 +8,8 @@ import ( "eth2-exporter/types" "eth2-exporter/utils" "fmt" - "io/ioutil" "log" + "os" "strings" "time" @@ -98,7 +98,7 @@ func VerifyReceipt(googleClient *playstore.Client, receipt *types.PremiumData) ( } func initGoogle() *playstore.Client { - jsonKey, err := ioutil.ReadFile(utils.Config.Frontend.AppSubsGoogleJSONPath) + jsonKey, err := os.ReadFile(utils.Config.Frontend.AppSubsGoogleJSONPath) if err != nil { log.Fatal(err) } diff --git a/exporter/rocketpool.go b/exporter/rocketpool.go index 05eabb370f..46556df4a7 100644 --- a/exporter/rocketpool.go +++ b/exporter/rocketpool.go @@ -3,7 +3,7 @@ package exporter import ( "encoding/json" "fmt" - "io/ioutil" + "io" "math" "math/big" "net/http" @@ -1858,7 +1858,7 @@ func DownloadRewardsFile(fileName string, interval uint64, cid string, isDaemon continue } else { // If we got here, we have a successful download - bytes, err := ioutil.ReadAll(resp.Body) + bytes, err := io.ReadAll(resp.Body) if err != nil { errBuilder.WriteString(fmt.Sprintf("Error reading response bytes from %s: %s\n", url, err.Error())) continue diff --git a/handlers/api.go b/handlers/api.go index f97110ce04..eef5a7eec0 100644 --- a/handlers/api.go +++ b/handlers/api.go @@ -15,7 +15,7 @@ import ( "eth2-exporter/types" "eth2-exporter/utils" "fmt" - "io/ioutil" + "io" "math" "math/big" "net/http" @@ -829,7 +829,7 @@ func ApiDashboard(w http.ResponseWriter, r *http.Request) { j := json.NewEncoder(w) - body, err := ioutil.ReadAll(r.Body) + body, err := io.ReadAll(r.Body) if err != nil { utils.LogError(err, "reading body", 0) sendErrorResponse(w, r.URL.String(), "could not read body") @@ -3508,7 +3508,7 @@ func clientStatsPost(w http.ResponseWriter, r *http.Request, apiKey, machine str return } - body, err := ioutil.ReadAll(r.Body) + body, err := io.ReadAll(r.Body) if err != nil { logger.Warnf("error reading body | err: %v", err) sendErrorResponse(w, r.URL.String(), "could not read body") diff --git a/handlers/stripe.go b/handlers/stripe.go index dafd6bf9db..af11dd67e4 100644 --- a/handlers/stripe.go +++ b/handlers/stripe.go @@ -11,7 +11,6 @@ import ( "fmt" "html/template" "io" - "io/ioutil" "net/http" "time" @@ -195,7 +194,7 @@ func StripeWebhook(w http.ResponseWriter, r *http.Request) { http.Error(w, http.StatusText(http.StatusMethodNotAllowed), http.StatusMethodNotAllowed) return } - b, err := ioutil.ReadAll(r.Body) + b, err := io.ReadAll(r.Body) if err != nil { http.Error(w, err.Error(), http.StatusBadRequest) logger.WithError(err).Error("error failed to read body for StripeWebhook") diff --git a/handlers/user.go b/handlers/user.go index a3310bfe79..d6415c1bc1 100644 --- a/handlers/user.go +++ b/handlers/user.go @@ -14,7 +14,6 @@ import ( "fmt" "html/template" "io" - "io/ioutil" "net/http" "net/url" "strconv" @@ -352,7 +351,7 @@ func RemoveAllValidatorsAndUnsubscribe(w http.ResponseWriter, r *http.Request) { user := getUser(r) - body, err := ioutil.ReadAll(r.Body) + body, err := io.ReadAll(r.Body) if err != nil { logger.Errorf("error reading body of request: %v, %v", r.URL.String(), err) ErrorOrJSONResponse(w, r, "Internal server error", http.StatusInternalServerError) @@ -382,7 +381,7 @@ func AddValidatorsAndSubscribe(w http.ResponseWriter, r *http.Request) { user := getUser(r) - body, err := ioutil.ReadAll(r.Body) + body, err := io.ReadAll(r.Body) if err != nil { logger.Errorf("error reading body of request: %v, %v", r.URL.String(), err) ErrorOrJSONResponse(w, r, "Internal server error", http.StatusInternalServerError) @@ -454,7 +453,7 @@ func UserUpdateSubscriptions(w http.ResponseWriter, r *http.Request) { user := getUser(r) - body, err := ioutil.ReadAll(r.Body) + body, err := io.ReadAll(r.Body) if err != nil { logger.Errorf("error reading body of request: %v, %v", r.URL.String(), err) ErrorOrJSONResponse(w, r, "Internal server error", http.StatusInternalServerError) @@ -1620,7 +1619,7 @@ func UserDashboardWatchlistAdd(w http.ResponseWriter, r *http.Request) { SetAutoContentType(w, r) //w.Header().Set("Content-Type", "text/html") user := getUser(r) - body, err := ioutil.ReadAll(r.Body) + body, err := io.ReadAll(r.Body) if err != nil { logger.Errorf("error reading body of request: %v, %v", r.URL.String(), err) ErrorOrJSONResponse(w, r, "Internal server error", http.StatusInternalServerError) @@ -1790,7 +1789,7 @@ func MultipleUsersNotificationsSubscribeWeb(w http.ResponseWriter, r *http.Reque } var jsonObjects []SubIntent - b, err := ioutil.ReadAll(r.Body) + b, err := io.ReadAll(r.Body) if err != nil { logger.Errorf("error reading body %v URL: %v", err, r.URL.String()) sendErrorResponse(w, r.URL.String(), "could not parse body") diff --git a/rpc/lighthouse.go b/rpc/lighthouse.go index 2f8cabaf7d..0775bd2e9e 100644 --- a/rpc/lighthouse.go +++ b/rpc/lighthouse.go @@ -8,7 +8,7 @@ import ( "eth2-exporter/types" "eth2-exporter/utils" "fmt" - "io/ioutil" + "io" "math/big" "net/http" "strconv" @@ -891,7 +891,7 @@ func (lc *LighthouseClient) get(url string) ([]byte, error) { defer resp.Body.Close() - data, err := ioutil.ReadAll(resp.Body) + data, err := io.ReadAll(resp.Body) if resp.StatusCode != http.StatusOK { if resp.StatusCode == http.StatusNotFound { diff --git a/services/notifications.go b/services/notifications.go index f8784d12ae..183e72e827 100644 --- a/services/notifications.go +++ b/services/notifications.go @@ -2336,7 +2336,7 @@ func collectMonitoringMachine( rowKeys := gcp_bigtable.RowList{} for _, data := range allSubscribed { - rowKeys = append(rowKeys, db.GetMachineRowKey(data.UserID, "system", data.MachineName)) + rowKeys = append(rowKeys, db.BigtableClient.GetMachineRowKey(data.UserID, "system", data.MachineName)) } machineDataOfSubscribed, err := db.BigtableClient.GetMachineMetricsForNotifications(rowKeys) diff --git a/utils/oauthutils.go b/utils/oauthutils.go index 89364be64e..8407f678f3 100644 --- a/utils/oauthutils.go +++ b/utils/oauthutils.go @@ -5,7 +5,7 @@ import ( "encoding/json" "errors" "fmt" - "io/ioutil" + "io" "net/http" "strings" "time" @@ -241,7 +241,7 @@ func AuthorizedAPIMiddleware(next http.Handler) http.Handler { // as context to the request. By doing this here, // we can use base.FormValueOrJson(key) without multiple parsings of the same body if r.Method == "POST" && r.Header.Get("Content-Type") == "application/json" { - body, err := ioutil.ReadAll(r.Body) + body, err := io.ReadAll(r.Body) if err == nil { keyVal := make(map[string]interface{}) json.Unmarshal(body, &keyVal) diff --git a/utils/utils.go b/utils/utils.go index e79e89e3bd..87e165b944 100644 --- a/utils/utils.go +++ b/utils/utils.go @@ -17,7 +17,7 @@ import ( "fmt" "html/template" "image/color" - "io/ioutil" + "io" "log" "math" "math/big" @@ -244,7 +244,7 @@ func GetTemplateFuncs() template.FuncMap { // IncludeHTML adds html to the page func IncludeHTML(path string) template.HTML { - b, err := ioutil.ReadFile(path) + b, err := os.ReadFile(path) if err != nil { log.Printf("includeHTML - error reading file: %v", err) return "" @@ -931,7 +931,7 @@ func ValidateReCAPTCHA(recaptchaResponse string) (bool, error) { return false, err } defer req.Body.Close() - body, err := ioutil.ReadAll(req.Body) // Read the response from Google + body, err := io.ReadAll(req.Body) // Read the response from Google if err != nil { return false, err } @@ -992,7 +992,7 @@ func TryFetchContractMetadata(address []byte) (*types.ContractMetadata, error) { // } // if resp.StatusCode == 200 { -// body, err := ioutil.ReadAll(resp.Body) +// body, err := io.ReadAll(resp.Body) // if err != nil { // return nil, err // } @@ -1067,7 +1067,7 @@ func getABIFromEtherscan(address []byte) (*types.ContractMetadata, error) { return nil, fmt.Errorf("StatusCode: '%d', Status: '%s'", resp.StatusCode, resp.Status) } - body, err := ioutil.ReadAll(resp.Body) + body, err := io.ReadAll(resp.Body) if err != nil { return nil, err } From 3267c467fd61deefb9370eba70a7cfd2fdb5b182 Mon Sep 17 00:00:00 2001 From: peter <1674920+peterbitfly@users.noreply.github.com> Date: Thu, 31 Aug 2023 17:37:49 +0200 Subject: [PATCH 052/106] (BIDS-2379) address cr issues --- db/statistics.go | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/db/statistics.go b/db/statistics.go index e800e56608..2fad68c988 100644 --- a/db/statistics.go +++ b/db/statistics.go @@ -11,7 +11,6 @@ import ( "fmt" "math/big" "strings" - "sync" "time" "github.com/ethereum/go-ethereum/common" @@ -1091,9 +1090,7 @@ func WriteValidatorFailedAttestationsStatisticsForDay(validators []uint64, day u logrus.Infof("exporting 'failed attestations' statistics firstEpoch: %v lastEpoch: %v", firstEpoch, lastEpoch) - // first key is the batch start index and the second is the validator id validatorMap := map[uint64]*types.ValidatorMissedAttestationsStatistic{} - mux := sync.Mutex{} batchSize := 10000 for i := 0; i < len(validators); i += batchSize { @@ -1109,11 +1106,9 @@ func WriteValidatorFailedAttestationsStatisticsForDay(validators []uint64, day u if err != nil { return fmt.Errorf("error in GetValidatorMissedAttestationsCount for fromEpoch [%v] and toEpoch [%v]: %w", firstEpoch, lastEpoch, err) } - mux.Lock() for validator, stats := range ma { validatorMap[validator] = stats } - mux.Unlock() } logrus.Infof("fetching 'failed attestations' done in %v, now we export them to the db", time.Since(start)) @@ -1126,7 +1121,7 @@ func WriteValidatorFailedAttestationsStatisticsForDay(validators []uint64, day u g, gCtx := errgroup.WithContext(ctx) g.SetLimit(50) - dbBatchSize := 100 // max: 65535 / 4, but we are faster with smaller batches + dbBatchSize := 100 for b := 0; b < len(maArr); b += dbBatchSize { start := b From be83f206de3641d6a3a7cf89e05487a89609407e Mon Sep 17 00:00:00 2001 From: peter <1674920+peterbitfly@users.noreply.github.com> Date: Thu, 31 Aug 2023 17:40:43 +0200 Subject: [PATCH 053/106] (BIDS-2379) update ci to go v1.20 --- .github/workflows/build.yml | 4 ++-- .github/workflows/staticcheck.yml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index be1dfa79da..27fe09600c 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -14,10 +14,10 @@ jobs: name: Build runs-on: ubuntu-latest steps: - - name: Set up Go 1.18 + - name: Set up Go 1.20 uses: actions/setup-go@v3 with: - go-version: 1.18 + go-version: 1.20 id: go - name: Check out code into the Go module directory diff --git a/.github/workflows/staticcheck.yml b/.github/workflows/staticcheck.yml index 10f7af1b21..9ea940d850 100644 --- a/.github/workflows/staticcheck.yml +++ b/.github/workflows/staticcheck.yml @@ -17,7 +17,7 @@ jobs: fail-fast: false matrix: os: ["ubuntu-latest"] - go: ["1.18.x"] + go: ["1.20.x"] runs-on: ${{ matrix.os }} steps: - uses: actions/checkout@v1 From 8a1caa463174a38f68e806090016a353183868dc Mon Sep 17 00:00:00 2001 From: peter <1674920+peterbitfly@users.noreply.github.com> Date: Thu, 31 Aug 2023 17:45:59 +0200 Subject: [PATCH 054/106] (BIDS-2379) fix go vet --- db/bigtable.go | 86 +++++++++++++++++++++++++------------------------- 1 file changed, 43 insertions(+), 43 deletions(-) diff --git a/db/bigtable.go b/db/bigtable.go index 68af68d0cb..e43cd9fd26 100644 --- a/db/bigtable.go +++ b/db/bigtable.go @@ -573,63 +573,63 @@ func (bigtable *Bigtable) SaveSyncCommitteesAssignments(startSlot, endSlot uint6 return nil //disabled as not needed - ctx, cancel := context.WithTimeout(context.Background(), time.Minute*10) - defer cancel() + // ctx, cancel := context.WithTimeout(context.Background(), time.Minute*10) + // defer cancel() - start := time.Now() - ts := gcp_bigtable.Timestamp(0) + // start := time.Now() + // ts := gcp_bigtable.Timestamp(0) - muts := make([]*gcp_bigtable.Mutation, 0, MAX_BATCH_MUTATIONS) - keys := make([]string, 0, MAX_BATCH_MUTATIONS) + // muts := make([]*gcp_bigtable.Mutation, 0, MAX_BATCH_MUTATIONS) + // keys := make([]string, 0, MAX_BATCH_MUTATIONS) - for i := startSlot; i <= endSlot; i++ { - for _, validator := range validators { - mut := gcp_bigtable.NewMutation() - mut.Set(SYNC_COMMITTEES_FAMILY, "s", ts, []byte{}) + // for i := startSlot; i <= endSlot; i++ { + // for _, validator := range validators { + // mut := gcp_bigtable.NewMutation() + // mut.Set(SYNC_COMMITTEES_FAMILY, "s", ts, []byte{}) - key := fmt.Sprintf("%s:%s:%s:%s", bigtable.chainId, bigtable.validatorIndexToKey(validator), bigtable.reversedPaddedEpoch(utils.EpochOfSlot(i)), bigtable.reversedPaddedSlot(i)) + // key := fmt.Sprintf("%s:%s:%s:%s", bigtable.chainId, bigtable.validatorIndexToKey(validator), bigtable.reversedPaddedEpoch(utils.EpochOfSlot(i)), bigtable.reversedPaddedSlot(i)) - muts = append(muts, mut) - keys = append(keys, key) + // muts = append(muts, mut) + // keys = append(keys, key) - if len(muts) == MAX_BATCH_MUTATIONS { - logger.Infof("saving %v mutations for sync duties", len(muts)) - errs, err := bigtable.tableValidatorSyncCommittees.ApplyBulk(ctx, keys, muts) + // if len(muts) == MAX_BATCH_MUTATIONS { + // logger.Infof("saving %v mutations for sync duties", len(muts)) + // errs, err := bigtable.tableValidatorSyncCommittees.ApplyBulk(ctx, keys, muts) - if err != nil { - return err - } + // if err != nil { + // return err + // } - for _, err := range errs { - if err != nil { - return err - } - } + // for _, err := range errs { + // if err != nil { + // return err + // } + // } - muts = make([]*gcp_bigtable.Mutation, 0, MAX_BATCH_MUTATIONS) - keys = make([]string, 0, MAX_BATCH_MUTATIONS) - } - } + // muts = make([]*gcp_bigtable.Mutation, 0, MAX_BATCH_MUTATIONS) + // keys = make([]string, 0, MAX_BATCH_MUTATIONS) + // } + // } - } + // } - if len(muts) > 0 { - logger.Infof("saving %v mutations for sync duties", len(muts)) - errs, err := bigtable.tableValidatorSyncCommittees.ApplyBulk(ctx, keys, muts) + // if len(muts) > 0 { + // logger.Infof("saving %v mutations for sync duties", len(muts)) + // errs, err := bigtable.tableValidatorSyncCommittees.ApplyBulk(ctx, keys, muts) - if err != nil { - return err - } + // if err != nil { + // return err + // } - for _, err := range errs { - if err != nil { - return err - } - } - } + // for _, err := range errs { + // if err != nil { + // return err + // } + // } + // } - logger.Infof("exported sync committee assignments to bigtable in %v", time.Since(start)) - return nil + // logger.Infof("exported sync committee assignments to bigtable in %v", time.Since(start)) + // return nil } func (bigtable *Bigtable) SaveAttestations(blocks map[uint64]map[string]*types.Block) error { From d7b8ffcf3d99bd459246fc9f2572a10062b222db Mon Sep 17 00:00:00 2001 From: peter <1674920+peterbitfly@users.noreply.github.com> Date: Thu, 31 Aug 2023 18:30:25 +0200 Subject: [PATCH 055/106] (BIDS-2379) fix underflow when loading epoch reward details --- db/bigtable.go | 52 ++++++++++++++----- handlers/validator.go | 45 ++++++++++------ local-deployment/provision-explorer-config.sh | 1 + 3 files changed, 69 insertions(+), 29 deletions(-) diff --git a/db/bigtable.go b/db/bigtable.go index e43cd9fd26..e621b7e096 100644 --- a/db/bigtable.go +++ b/db/bigtable.go @@ -25,15 +25,16 @@ import ( var BigtableClient *Bigtable const ( - DEFAULT_FAMILY = "f" - VALIDATOR_BALANCES_FAMILY = "vb" - ATTESTATIONS_FAMILY = "at" - PROPOSALS_FAMILY = "pr" - SYNC_COMMITTEES_FAMILY = "sc" - INCOME_DETAILS_COLUMN_FAMILY = "id" - STATS_COLUMN_FAMILY = "stats" - MACHINE_METRICS_COLUMN_FAMILY = "mm" - SERIES_FAMILY = "series" + DEFAULT_FAMILY = "f" + VALIDATOR_BALANCES_FAMILY = "vb" + ATTESTATIONS_FAMILY = "at" + PROPOSALS_FAMILY = "pr" + SYNC_COMMITTEES_FAMILY = "sc" + SYNC_COMMITTEES_PARTICIPATION_FAMILY = "sp" + INCOME_DETAILS_COLUMN_FAMILY = "id" + STATS_COLUMN_FAMILY = "stats" + MACHINE_METRICS_COLUMN_FAMILY = "mm" + SERIES_FAMILY = "series" SUM_COLUMN = "sum" @@ -850,14 +851,16 @@ func (bigtable *Bigtable) SaveSyncComitteeDuties(blocks map[uint64]map[string]*t return nil } - muts := make([]*gcp_bigtable.Mutation, 0, MAX_BATCH_MUTATIONS) - keys := make([]string, 0, MAX_BATCH_MUTATIONS) + muts := make([]*gcp_bigtable.Mutation, 0, utils.Config.Chain.Config.SlotsPerEpoch*utils.Config.Chain.Config.SyncCommitteeSize+1) + keys := make([]string, 0, utils.Config.Chain.Config.SlotsPerEpoch*utils.Config.Chain.Config.SyncCommitteeSize+1) for slot, validators := range dutiesBySlot { + participation := uint64(0) for validator, participated := range validators { mut := gcp_bigtable.NewMutation() if participated { mut.Set(SYNC_COMMITTEES_FAMILY, "s", gcp_bigtable.Timestamp((MAX_BLOCK_NUMBER-slot)*1000), []byte{}) + participation++ } else { mut.Set(SYNC_COMMITTEES_FAMILY, "s", gcp_bigtable.Timestamp(0), []byte{}) } @@ -865,7 +868,13 @@ func (bigtable *Bigtable) SaveSyncComitteeDuties(blocks map[uint64]map[string]*t muts = append(muts, mut) keys = append(keys, key) } - + mut := gcp_bigtable.NewMutation() + key := fmt.Sprintf("%s:%s:%s", bigtable.chainId, SYNC_COMMITTEES_PARTICIPATION_FAMILY, bigtable.reversedPaddedSlot(slot)) + participationEncoded := make([]byte, 8) + binary.LittleEndian.PutUint64(participationEncoded, uint64(participation)) + mut.Set(SYNC_COMMITTEES_PARTICIPATION_FAMILY, "s", gcp_bigtable.Timestamp(0), participationEncoded) + muts = append(muts, mut) + keys = append(keys, key) } errs, err := bigtable.tableValidatorSyncCommittees.ApplyBulk(ctx, keys, muts) @@ -1151,6 +1160,25 @@ func (bigtable *Bigtable) GetLastAttestationSlots(validators []uint64) (map[uint return res, nil } +func (bigtable *Bigtable) GetSyncParticipationBySlot(slot uint64) (uint64, error) { + + ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(time.Minute*5)) + defer cancel() + + key := fmt.Sprintf("%s:%s:%s", bigtable.chainId, SYNC_COMMITTEES_PARTICIPATION_FAMILY, bigtable.reversedPaddedSlot(slot)) + + row, err := bigtable.tableValidatorSyncCommittees.ReadRow(ctx, key) + if err != nil { + return 0, err + } + + for _, ri := range row[SYNC_COMMITTEES_PARTICIPATION_FAMILY] { + return binary.LittleEndian.Uint64(ri.Value), nil + } + + return 0, nil +} + func (bigtable *Bigtable) GetValidatorMissedAttestationHistory(validators []uint64, startEpoch uint64, endEpoch uint64) (map[uint64]map[uint64]bool, error) { if len(validators) == 0 { diff --git a/handlers/validator.go b/handlers/validator.go index e95527a385..5849248f6e 100644 --- a/handlers/validator.go +++ b/handlers/validator.go @@ -1577,7 +1577,6 @@ func ValidatorHistory(w http.ResponseWriter, r *http.Request) { } tableData := make([][]interface{}, 0) - if postExitEpochs > 0 { startEpoch := currentEpoch + 1 endEpoch := startEpoch + postExitEpochs @@ -1588,6 +1587,10 @@ func ValidatorHistory(w http.ResponseWriter, r *http.Request) { // if there are additional epochs with duties we have to go through all of them as there can be gaps (after the exit before the duty) for i := endEpoch; i >= startEpoch; i-- { + if i > endEpoch { + break + } + if incomeDetails[index] == nil || incomeDetails[index][i] == nil { continue } @@ -1618,6 +1621,11 @@ func ValidatorHistory(w http.ResponseWriter, r *http.Request) { } for epoch := endEpoch; epoch >= startEpoch && len(tableData) < pageLength; epoch-- { + + if epoch > endEpoch { + break + } + if incomeDetails[index] == nil || incomeDetails[index][epoch] == nil { if epoch <= endEpoch { rewardsStr := "pending..." @@ -1636,6 +1644,11 @@ func ValidatorHistory(w http.ResponseWriter, r *http.Request) { continue } tableData = append(tableData, icomeToTableData(epoch, incomeDetails[index][epoch], withdrawalMap[epoch], currency)) + + if epoch == 0 { + break + } + logger.Info(epoch) } } @@ -1938,14 +1951,14 @@ func ValidatorSync(w http.ResponseWriter, r *http.Request) { default: } - syncDuties, err := db.BigtableClient.GetValidatorSyncDutiesHistory([]uint64{}, epoch, epoch) + syncDuties, err := db.BigtableClient.GetValidatorSyncDutiesHistory([]uint64{validatorIndex}, epoch, epoch) if err != nil { return fmt.Errorf("error retrieving validator [%v] sync duty data from bigtable for epoch [%v]: %w", validatorIndex, epoch, err) } else if uint64(len(syncDuties[validatorIndex]))%utils.Config.Chain.Config.SlotsPerEpoch > 0 { return fmt.Errorf("wrong number [%v] of syncDuties for validator [%v] received from from bigtable for epoch [%v]", len(syncDuties[validatorIndex]), validatorIndex, epoch) } - for validator, duties := range syncDuties { + for _, duties := range syncDuties { for _, duty := range duties { mux.Lock() pageSlot, ok := pageSlotsMap[duty.Slot] @@ -1954,23 +1967,21 @@ func ValidatorSync(w http.ResponseWriter, r *http.Request) { continue // this is not the slot we are looking for } - // We want to know how many validators successfully participated in this duty - if duty.Status == 1 { - pageSlot.Participation++ + participation, err := db.BigtableClient.GetSyncParticipationBySlot(duty.Slot) + if err != nil { + return fmt.Errorf("error retrieving sync participation for slot %v", duty.Slot) + } + pageSlot.Participation = participation + + slotTime := utils.SlotToTime(duty.Slot) + if duty.Status == 0 && time.Since(slotTime) <= time.Minute { + duty.Status = 2 // scheduled } - // Get the status if the duty belongs to our validator - if validator == validatorIndex { - slotTime := utils.SlotToTime(duty.Slot) - if duty.Status == 0 && time.Since(slotTime) <= time.Minute { - duty.Status = 2 // scheduled - } - - if duty.Status == 0 { - missedSyncSlots = append(missedSyncSlots, duty.Slot) - } - pageSlot.Status = duty.Status + if duty.Status == 0 { + missedSyncSlots = append(missedSyncSlots, duty.Slot) } + pageSlot.Status = duty.Status pageSlotsMap[duty.Slot] = pageSlot mux.Unlock() } diff --git a/local-deployment/provision-explorer-config.sh b/local-deployment/provision-explorer-config.sh index 613c0c422b..7d42d42334 100644 --- a/local-deployment/provision-explorer-config.sh +++ b/local-deployment/provision-explorer-config.sh @@ -93,6 +93,7 @@ BIGTABLE_EMULATOR_HOST=$HOST cbt --project $PROJECT --instance $INSTANCE createt BIGTABLE_EMULATOR_HOST=$HOST cbt --project $PROJECT --instance $INSTANCE createfamily beaconchain_validator_proposals pr BIGTABLE_EMULATOR_HOST=$HOST cbt --project $PROJECT --instance $INSTANCE createtable beaconchain_validator_sync BIGTABLE_EMULATOR_HOST=$HOST cbt --project $PROJECT --instance $INSTANCE createfamily beaconchain_validator_sync sc +BIGTABLE_EMULATOR_HOST=$HOST cbt --project $PROJECT --instance $INSTANCE createfamily beaconchain_validator_sync sp BIGTABLE_EMULATOR_HOST=$HOST cbt --project $PROJECT --instance $INSTANCE createtable beaconchain_validator_income BIGTABLE_EMULATOR_HOST=$HOST cbt --project $PROJECT --instance $INSTANCE createfamily beaconchain_validator_income id BIGTABLE_EMULATOR_HOST=$HOST cbt --project $PROJECT --instance $INSTANCE createfamily beaconchain_validator_income stats From 83cb15377ab72e22af26f682d1a7bf6ba656e862 Mon Sep 17 00:00:00 2001 From: peter <1674920+peterbitfly@users.noreply.github.com> Date: Thu, 31 Aug 2023 18:30:42 +0200 Subject: [PATCH 056/106] (BIDS-2379) remove debug log --- handlers/validator.go | 1 - 1 file changed, 1 deletion(-) diff --git a/handlers/validator.go b/handlers/validator.go index 5849248f6e..0f003fe9ea 100644 --- a/handlers/validator.go +++ b/handlers/validator.go @@ -1648,7 +1648,6 @@ func ValidatorHistory(w http.ResponseWriter, r *http.Request) { if epoch == 0 { break } - logger.Info(epoch) } } From 7a203dc11234deb92e872b69051bc6d844cafa8f Mon Sep 17 00:00:00 2001 From: peter <1674920+peterbitfly@users.noreply.github.com> Date: Thu, 31 Aug 2023 19:32:40 +0200 Subject: [PATCH 057/106] (BIDS-2379) please linter --- services/configuration.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/services/configuration.go b/services/configuration.go index df89548989..1423149236 100644 --- a/services/configuration.go +++ b/services/configuration.go @@ -16,10 +16,12 @@ const ( ConfigurationKeyHardforkName types.ExplorerConfigurationKey = "HardforkName" ) -/*** +/* +** This is the list of possible configurations that can be changed in the explorer administration Per default these values will be taken, overridden by the values from the db, if they exist. -***/ +** +*/ var DefaultExplorerConfiguration types.ExplorerConfigurationMap = types.ExplorerConfigurationMap{ ConfigurationCategorySlotViz: { ConfigurationKeyVisibleFromEpoch: {Value: "0", DataType: "int"}, From 134da85fe4f0a096c45e8823d9a33612aced4347 Mon Sep 17 00:00:00 2001 From: peter <1674920+peterbitfly@users.noreply.github.com> Date: Thu, 31 Aug 2023 19:34:54 +0200 Subject: [PATCH 058/106] (BIDS-2379) update ci --- .github/workflows/staticcheck.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/staticcheck.yml b/.github/workflows/staticcheck.yml index 9ea940d850..2a985cd67f 100644 --- a/.github/workflows/staticcheck.yml +++ b/.github/workflows/staticcheck.yml @@ -31,6 +31,6 @@ jobs: - run: "go vet ./..." - uses: dominikh/staticcheck-action@v1.2.0 with: - version: "2022.1.1" + version: "2023.1.5" install-go: false cache-key: ${{ matrix.go }} \ No newline at end of file From ef3abbe0dcdf44bf3f00d86d7fadb51d09b463b9 Mon Sep 17 00:00:00 2001 From: peter <1674920+peterbitfly@users.noreply.github.com> Date: Thu, 31 Aug 2023 19:36:43 +0200 Subject: [PATCH 059/106] (BIDS-2379) update ci --- .github/workflows/build.yml | 8 -------- 1 file changed, 8 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 27fe09600c..10799841b6 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -23,14 +23,6 @@ jobs: - name: Check out code into the Go module directory uses: actions/checkout@v3 - - name: Check gofmt - run: | - nonformatted="$(gofmt -l . 2>&1)" - [ -z "$nonformatted" ] && exit 0 - echo "Following files need to be properly formatted:" - echo "$nonformatted" - exit 1 - - name: Get dependencies run: | go install github.com/swaggo/swag/cmd/swag@v1.7.0 && swag init --parseDependency --parseInternal --parseDepth 1 -g handlers/api.go From 4b48e0590ec9edbba6a7fd011d478436a022cb07 Mon Sep 17 00:00:00 2001 From: peter <1674920+peterbitfly@users.noreply.github.com> Date: Thu, 31 Aug 2023 19:40:56 +0200 Subject: [PATCH 060/106] (BIDS-2379) update ci --- .github/workflows/build.yml | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 10799841b6..4d6df738e5 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -15,14 +15,22 @@ jobs: runs-on: ubuntu-latest steps: - name: Set up Go 1.20 - uses: actions/setup-go@v3 + uses: actions/setup-go@v4 with: - go-version: 1.20 + go-version: '1.20' id: go - name: Check out code into the Go module directory uses: actions/checkout@v3 + - name: Check gofmt + run: | + nonformatted="$(gofmt -l . 2>&1)" + [ -z "$nonformatted" ] && exit 0 + echo "Following files need to be properly formatted:" + echo "$nonformatted" + exit 1 + - name: Get dependencies run: | go install github.com/swaggo/swag/cmd/swag@v1.7.0 && swag init --parseDependency --parseInternal --parseDepth 1 -g handlers/api.go From d74388f2ed697e12d8d633c8a0847844ce7be891 Mon Sep 17 00:00:00 2001 From: peter <1674920+peterbitfly@users.noreply.github.com> Date: Thu, 31 Aug 2023 19:42:25 +0200 Subject: [PATCH 061/106] (BIDS-2379) update ci --- .github/workflows/build.yml | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 4d6df738e5..36eb91f84d 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -14,14 +14,17 @@ jobs: name: Build runs-on: ubuntu-latest steps: + - name: Check out code into the Go module directory + uses: actions/checkout@v3 + - name: Set up Go 1.20 uses: actions/setup-go@v4 with: - go-version: '1.20' + go-version-file: './go.mod' id: go - - name: Check out code into the Go module directory - uses: actions/checkout@v3 + - name: Output go version + run: go version - name: Check gofmt run: | From a1535ccf4d55974c87f948d3ec18cf81957f9ad9 Mon Sep 17 00:00:00 2001 From: peter <1674920+peterbitfly@users.noreply.github.com> Date: Thu, 31 Aug 2023 19:52:41 +0200 Subject: [PATCH 062/106] (BIDS-2379) store max validator index with a balance at a given epoch --- db/bigtable.go | 53 +++++++++++++++++++++++++++++++++++++++----------- 1 file changed, 42 insertions(+), 11 deletions(-) diff --git a/db/bigtable.go b/db/bigtable.go index e621b7e096..e349931491 100644 --- a/db/bigtable.go +++ b/db/bigtable.go @@ -25,16 +25,17 @@ import ( var BigtableClient *Bigtable const ( - DEFAULT_FAMILY = "f" - VALIDATOR_BALANCES_FAMILY = "vb" - ATTESTATIONS_FAMILY = "at" - PROPOSALS_FAMILY = "pr" - SYNC_COMMITTEES_FAMILY = "sc" - SYNC_COMMITTEES_PARTICIPATION_FAMILY = "sp" - INCOME_DETAILS_COLUMN_FAMILY = "id" - STATS_COLUMN_FAMILY = "stats" - MACHINE_METRICS_COLUMN_FAMILY = "mm" - SERIES_FAMILY = "series" + DEFAULT_FAMILY = "f" + VALIDATOR_BALANCES_FAMILY = "vb" + VALIDATOR_HIGHEST_ACTIVE_INDEX_FAMILY = "ha" + ATTESTATIONS_FAMILY = "at" + PROPOSALS_FAMILY = "pr" + SYNC_COMMITTEES_FAMILY = "sc" + SYNC_COMMITTEES_PARTICIPATION_FAMILY = "sp" + INCOME_DETAILS_COLUMN_FAMILY = "id" + STATS_COLUMN_FAMILY = "stats" + MACHINE_METRICS_COLUMN_FAMILY = "mm" + SERIES_FAMILY = "series" SUM_COLUMN = "sum" @@ -421,8 +422,14 @@ func (bigtable *Bigtable) SaveValidatorBalances(epoch uint64, validators []*type muts := make([]*gcp_bigtable.Mutation, 0, MAX_BATCH_MUTATIONS) keys := make([]string, 0, MAX_BATCH_MUTATIONS) + highestActiveIndex := uint64(0) epochKey := bigtable.reversedPaddedEpoch(epoch) for _, validator := range validators { + + if validator.Balance > 0 && validator.Index > highestActiveIndex { + highestActiveIndex = validator.Index + } + balanceEncoded := make([]byte, 8) binary.LittleEndian.PutUint64(balanceEncoded, validator.Balance) @@ -452,6 +459,12 @@ func (bigtable *Bigtable) SaveValidatorBalances(epoch uint64, validators []*type } } + highestActiveIndexEncoded := make([]byte, 8) + binary.LittleEndian.PutUint64(highestActiveIndexEncoded, highestActiveIndex) + + mut := &gcp_bigtable.Mutation{} + mut.Set(VALIDATOR_HIGHEST_ACTIVE_INDEX_FAMILY, VALIDATOR_HIGHEST_ACTIVE_INDEX_FAMILY, ts, highestActiveIndexEncoded) + if len(muts) > 0 { errs, err := bigtable.tableValidatorBalances.ApplyBulk(ctx, keys, muts) @@ -464,6 +477,12 @@ func (bigtable *Bigtable) SaveValidatorBalances(epoch uint64, validators []*type } } + key := fmt.Sprintf("%s:%s:%s", bigtable.chainId, VALIDATOR_HIGHEST_ACTIVE_INDEX_FAMILY, epochKey) + err := bigtable.tableValidatorBalances.Apply(ctx, key, mut) + if err != nil { + return err + } + // logger.Infof("exported validator balances to bigtable in %v", time.Since(start)) return nil } @@ -894,7 +913,19 @@ func (bigtable *Bigtable) SaveSyncComitteeDuties(blocks map[uint64]map[string]*t // GetMaxValidatorindexForEpoch returns the higest validatorindex with a balance at that epoch func (bigtable *Bigtable) GetMaxValidatorindexForEpoch(epoch uint64) (uint64, error) { - // TODO: Implement + ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(time.Minute*5)) + defer cancel() + + key := fmt.Sprintf("%s:%s:%s", bigtable.chainId, VALIDATOR_HIGHEST_ACTIVE_INDEX_FAMILY, bigtable.reversedPaddedEpoch(epoch)) + + row, err := bigtable.tableValidatorBalances.ReadRow(ctx, key) + if err != nil { + return 0, err + } + + for _, ri := range row[VALIDATOR_HIGHEST_ACTIVE_INDEX_FAMILY] { + return binary.LittleEndian.Uint64(ri.Value), nil + } return 0, nil } From ff41a061d446064b82da174c738aa5547d44af39 Mon Sep 17 00:00:00 2001 From: peter <1674920+peterbitfly@users.noreply.github.com> Date: Thu, 31 Aug 2023 20:18:20 +0200 Subject: [PATCH 063/106] (BIDS-2379) improve docker build speed, allow to specify evm trace mode (parity or geth or both) --- cmd/eth1indexer/main.go | 11 +- local-deployment/docker-compose.yml | 32 +++-- local-deployment/provision-explorer-config.sh | 1 + rpc/erigon.go | 114 ++++++++++-------- 4 files changed, 94 insertions(+), 64 deletions(-) diff --git a/cmd/eth1indexer/main.go b/cmd/eth1indexer/main.go index 6399313ded..0a82fb13ce 100644 --- a/cmd/eth1indexer/main.go +++ b/cmd/eth1indexer/main.go @@ -46,6 +46,7 @@ func main() { offsetBlocks := flag.Int64("blocks.offset", 100, "Blocks offset") checkBlocksGaps := flag.Bool("blocks.gaps", false, "Check for gaps in the blocks table") checkBlocksGapsLookback := flag.Int("blocks.gaps.lookback", 1000000, "Lookback for gaps check of the blocks table") + traceMode := flag.String("blocks.tracemode", "parity/geth", "Trace mode to use, can bei either 'parity', 'geth' or 'parity/geth' for both") concurrencyData := flag.Int64("data.concurrency", 30, "Concurrency to use when indexing data from bigtable") startData := flag.Int64("data.start", 0, "Block to start indexing") @@ -216,7 +217,7 @@ func main() { cache := freecache.NewCache(100 * 1024 * 1024) // 100 MB limit if *block != 0 { - err = IndexFromNode(bt, client, *block, *block, *concurrencyBlocks) + err = IndexFromNode(bt, client, *block, *block, *concurrencyBlocks, *traceMode) if err != nil { logrus.WithError(err).Fatalf("error indexing from node, start: %v end: %v concurrency: %v", *block, *block, *concurrencyBlocks) } @@ -241,7 +242,7 @@ func main() { } if *endBlocks != 0 && *startBlocks < *endBlocks { - err = IndexFromNode(bt, client, *startBlocks, *endBlocks, *concurrencyBlocks) + err = IndexFromNode(bt, client, *startBlocks, *endBlocks, *concurrencyBlocks, *traceMode) if err != nil { logrus.WithError(err).Fatalf("error indexing from node, start: %v end: %v concurrency: %v", *startBlocks, *endBlocks, *concurrencyBlocks) } @@ -311,7 +312,7 @@ func main() { endBlock = int64(lastBlockFromNode) } - err = IndexFromNode(bt, client, startBlock, endBlock, *concurrencyBlocks) + err = IndexFromNode(bt, client, startBlock, endBlock, *concurrencyBlocks, *traceMode) if err != nil { errMsg := "error indexing from node" errFields := map[string]interface{}{ @@ -723,7 +724,7 @@ func ProcessMetadataUpdates(bt *db.Bigtable, client *rpc.ErigonClient, prefix st // } } -func IndexFromNode(bt *db.Bigtable, client *rpc.ErigonClient, start, end, concurrency int64) error { +func IndexFromNode(bt *db.Bigtable, client *rpc.ErigonClient, start, end, concurrency int64, traceMode string) error { ctx := context.Background() g, gCtx := errgroup.WithContext(ctx) g.SetLimit(int(concurrency)) @@ -744,7 +745,7 @@ func IndexFromNode(bt *db.Bigtable, client *rpc.ErigonClient, start, end, concur } blockStartTs := time.Now() - bc, timings, err := client.GetBlock(i) + bc, timings, err := client.GetBlock(i, traceMode) if err != nil { return fmt.Errorf("error getting block: %v from ethereum node err: %w", i, err) } diff --git a/local-deployment/docker-compose.yml b/local-deployment/docker-compose.yml index 04b964691f..f6ecc16411 100644 --- a/local-deployment/docker-compose.yml +++ b/local-deployment/docker-compose.yml @@ -1,7 +1,10 @@ version: '3' services: indexer: - build: ../ + build: + context: ../ + args: + - target=explorer command: ./explorer -config /app/config.yml volumes: - ./config.yml:/app/config.yml @@ -9,31 +12,46 @@ services: - INDEXER_ENABLED=true network_mode: "host" eth1indexer: - build: ../ - command: ./eth1indexer -config /app/config.yml -blocks.concurrency 1 -data.concurrency 1 --balances.enabled + build: + context: ../ + args: + - target=eth1indexer + command: ./eth1indexer -config /app/config.yml -blocks.concurrency 1 -blocks.tracemode 'geth' -data.concurrency 1 --balances.enabled volumes: - ./config.yml:/app/config.yml network_mode: "host" rewards-exporter: - build: ../ + build: + context: ../ + args: + - target=rewards-exporter command: ./rewards-exporter -config /app/config.yml volumes: - ./config.yml:/app/config.yml network_mode: "host" statistics: - build: ../ + build: + context: ../ + args: + - target=stats command: ./statistics -config /app/config.yml --charts.enabled --graffiti.enabled -validators.enabled volumes: - ./config.yml:/app/config.yml network_mode: "host" frontend-data-updater: - build: ../ + build: + context: ../ + args: + - target=frontend-data-updater command: ./frontend-data-updater -config /app/config.yml volumes: - ./config.yml:/app/config.yml network_mode: "host" frontend: - build: ../ + build: + context: ../ + args: + - target=explorer command: ./explorer -config /app/config.yml volumes: - ./config.yml:/app/config.yml diff --git a/local-deployment/provision-explorer-config.sh b/local-deployment/provision-explorer-config.sh index 7d42d42334..da2f9b59b0 100644 --- a/local-deployment/provision-explorer-config.sh +++ b/local-deployment/provision-explorer-config.sh @@ -87,6 +87,7 @@ HOST="127.0.0.1:$LBT_PORT" BIGTABLE_EMULATOR_HOST=$HOST cbt --project $PROJECT --instance $INSTANCE createtable beaconchain_validator_balances BIGTABLE_EMULATOR_HOST=$HOST cbt --project $PROJECT --instance $INSTANCE createfamily beaconchain_validator_balances vb +BIGTABLE_EMULATOR_HOST=$HOST cbt --project $PROJECT --instance $INSTANCE createfamily beaconchain_validator_balances ha BIGTABLE_EMULATOR_HOST=$HOST cbt --project $PROJECT --instance $INSTANCE createtable beaconchain_validator_attestations BIGTABLE_EMULATOR_HOST=$HOST cbt --project $PROJECT --instance $INSTANCE createfamily beaconchain_validator_attestations at BIGTABLE_EMULATOR_HOST=$HOST cbt --project $PROJECT --instance $INSTANCE createtable beaconchain_validator_proposals diff --git a/rpc/erigon.go b/rpc/erigon.go index 84d6577a6d..85c1bf1a4d 100644 --- a/rpc/erigon.go +++ b/rpc/erigon.go @@ -73,7 +73,7 @@ func (client *ErigonClient) GetRPCClient() *geth_rpc.Client { return client.rpcClient } -func (client *ErigonClient) GetBlock(number int64) (*types.Eth1Block, *types.GetBlockTimings, error) { +func (client *ErigonClient) GetBlock(number int64, traceMode string) (*types.Eth1Block, *types.GetBlockTimings, error) { ctx, cancel := context.WithTimeout(context.Background(), time.Second*10) defer cancel() @@ -194,10 +194,67 @@ func (client *ErigonClient) GetBlock(number int64) (*types.Eth1Block, *types.Get if block.NumberU64() == 0 { // genesis block is not traceable return nil } - traces, err := client.TraceParity(block.NumberU64()) - if err != nil { - logger.Errorf("error tracing block via parity style traces (%v), %v: %v", block.Number(), block.Hash(), err) + var traceError error + if traceMode == "parity" || traceMode == "parity/geth" { + traces, err := client.TraceParity(block.NumberU64()) + + if err != nil { + logger.Errorf("error tracing block via parity style traces (%v), %v: %v", block.Number(), block.Hash(), err) + traceError = err + } else { + for _, trace := range traces { + if trace.Type == "reward" { + continue + } + + if trace.TransactionHash == "" { + continue + } + + if trace.TransactionPosition >= len(c.Transactions) { + return fmt.Errorf("error transaction position %v out of range", trace.TransactionPosition) + } + + if trace.Error == "" { + c.Transactions[trace.TransactionPosition].Status = 1 + } else { + c.Transactions[trace.TransactionPosition].Status = 0 + c.Transactions[trace.TransactionPosition].ErrorMsg = trace.Error + } + + tracePb := &types.Eth1InternalTransaction{ + Type: trace.Type, + Path: fmt.Sprint(trace.TraceAddress), + } + + if tracePb.Type == "call" { + tracePb.Type = trace.Action.CallType + } + + if trace.Type == "create" { + tracePb.From = common.FromHex(trace.Action.From) + tracePb.To = common.FromHex(trace.Result.Address) + tracePb.Value = common.FromHex(trace.Action.Value) + } else if trace.Type == "suicide" { + tracePb.From = common.FromHex(trace.Action.Address) + tracePb.To = common.FromHex(trace.Action.RefundAddress) + tracePb.Value = common.FromHex(trace.Action.Balance) + } else if trace.Type == "call" { + tracePb.From = common.FromHex(trace.Action.From) + tracePb.To = common.FromHex(trace.Action.To) + tracePb.Value = common.FromHex(trace.Action.Value) + } else { + spew.Dump(trace) + logrus.Fatalf("unknown trace type %v in tx %v", trace.Type, trace.TransactionHash) + } + + c.Transactions[trace.TransactionPosition].Itx = append(c.Transactions[trace.TransactionPosition].Itx, tracePb) + } + } + } + + if traceMode == "geth" || (traceError != nil && traceMode == "parity/geth") { gethTraceData, err := client.TraceGeth(block.Hash()) @@ -205,7 +262,7 @@ func (client *ErigonClient) GetBlock(number int64) (*types.Eth1Block, *types.Get return fmt.Errorf("error tracing block via geth style traces (%v), %v: %v", block.Number(), block.Hash(), err) } - logger.Infof("retrieved %v calls via geth", len(gethTraceData)) + // logger.Infof("retrieved %v calls via geth", len(gethTraceData)) for _, trace := range gethTraceData { if trace.Error == "" { @@ -244,54 +301,7 @@ func (client *ErigonClient) GetBlock(number int64) (*types.Eth1Block, *types.Get timings.Traces = time.Since(start) // logrus.Infof("retrieved %v traces for %v txs", len(traces), len(c.Transactions)) - for _, trace := range traces { - if trace.Type == "reward" { - continue - } - - if trace.TransactionHash == "" { - continue - } - - if trace.TransactionPosition >= len(c.Transactions) { - return fmt.Errorf("error transaction position %v out of range", trace.TransactionPosition) - } - - if trace.Error == "" { - c.Transactions[trace.TransactionPosition].Status = 1 - } else { - c.Transactions[trace.TransactionPosition].Status = 0 - c.Transactions[trace.TransactionPosition].ErrorMsg = trace.Error - } - - tracePb := &types.Eth1InternalTransaction{ - Type: trace.Type, - Path: fmt.Sprint(trace.TraceAddress), - } - if tracePb.Type == "call" { - tracePb.Type = trace.Action.CallType - } - - if trace.Type == "create" { - tracePb.From = common.FromHex(trace.Action.From) - tracePb.To = common.FromHex(trace.Result.Address) - tracePb.Value = common.FromHex(trace.Action.Value) - } else if trace.Type == "suicide" { - tracePb.From = common.FromHex(trace.Action.Address) - tracePb.To = common.FromHex(trace.Action.RefundAddress) - tracePb.Value = common.FromHex(trace.Action.Balance) - } else if trace.Type == "call" { - tracePb.From = common.FromHex(trace.Action.From) - tracePb.To = common.FromHex(trace.Action.To) - tracePb.Value = common.FromHex(trace.Action.Value) - } else { - spew.Dump(trace) - logrus.Fatalf("unknown trace type %v in tx %v", trace.Type, trace.TransactionHash) - } - - c.Transactions[trace.TransactionPosition].Itx = append(c.Transactions[trace.TransactionPosition].Itx, tracePb) - } return nil }) From 173b6307184ed9eccca459bfecdc649bcf92b727 Mon Sep 17 00:00:00 2001 From: peter <1674920+peterbitfly@users.noreply.github.com> Date: Thu, 31 Aug 2023 20:22:37 +0200 Subject: [PATCH 064/106] (BIDS-2379) fix build errors --- cmd/misc/main.go | 2 +- handlers/eth1Block.go | 2 +- rpc/erigon.go | 7 ++++++- 3 files changed, 8 insertions(+), 3 deletions(-) diff --git a/cmd/misc/main.go b/cmd/misc/main.go index edf9dbf4cb..7669cbf2b6 100644 --- a/cmd/misc/main.go +++ b/cmd/misc/main.go @@ -539,7 +539,7 @@ func indexMissingBlocks(start uint64, end uint64, bt *db.Bigtable, client *rpc.E logrus.Infof("block [%v] not found so we need to index it", j) if _, err := db.BigtableClient.GetBlockFromBlocksTable(j); err != nil { logrus.Infof("could not load [%v] from blocks table so we need to fetch it from the node and save it", j) - bc, _, err := client.GetBlock(int64(j)) + bc, _, err := client.GetBlock(int64(j), "parity/geth") if err != nil { utils.LogError(err, fmt.Sprintf("error getting block: %v from ethereum node", j), 0) } diff --git a/handlers/eth1Block.go b/handlers/eth1Block.go index b690c30c62..b914a6e95c 100644 --- a/handlers/eth1Block.go +++ b/handlers/eth1Block.go @@ -119,7 +119,7 @@ func Eth1Block(w http.ResponseWriter, r *http.Request) { func GetExecutionBlockPageData(number uint64, limit int) (*types.Eth1BlockPageData, error) { block, err := db.BigtableClient.GetBlockFromBlocksTable(number) if diffToHead := int64(services.LatestEth1BlockNumber()) - int64(number); err != nil && diffToHead < 0 && diffToHead >= -5 { - block, _, err = rpc.CurrentErigonClient.GetBlock(int64(number)) + block, _, err = rpc.CurrentErigonClient.GetBlock(int64(number), "parity/geth") } if err != nil { return nil, err diff --git a/rpc/erigon.go b/rpc/erigon.go index 85c1bf1a4d..8a02437a3e 100644 --- a/rpc/erigon.go +++ b/rpc/erigon.go @@ -200,7 +200,12 @@ func (client *ErigonClient) GetBlock(number int64, traceMode string) (*types.Eth traces, err := client.TraceParity(block.NumberU64()) if err != nil { - logger.Errorf("error tracing block via parity style traces (%v), %v: %v", block.Number(), block.Hash(), err) + if traceMode == "parity" { + return fmt.Errorf("error tracing block via parity style traces (%v), %v: %v", block.Number(), block.Hash(), err) + } else { + logger.Errorf("error tracing block via parity style traces (%v), %v: %v", block.Number(), block.Hash(), err) + + } traceError = err } else { for _, trace := range traces { From 6d5a7b9f5cfbe85eeef91e8750e4863814755417 Mon Sep 17 00:00:00 2001 From: peter <1674920+peterbitfly@users.noreply.github.com> Date: Thu, 31 Aug 2023 20:25:35 +0200 Subject: [PATCH 065/106] (BIDS-2379) update bigtable config doc --- bigtable_config.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/bigtable_config.md b/bigtable_config.md index acbbf097ef..a3af5d0ace 100644 --- a/bigtable_config.md +++ b/bigtable_config.md @@ -13,6 +13,7 @@ Column families: ``` cbt -project $PROJECT -instance $INSTANCE createfamily beaconchain_validator_balances vb +cbt -project $PROJECT -instance $INSTANCE createfamily beaconchain_validator_balances ha ``` ---- @@ -53,6 +54,7 @@ Column families: ``` cbt -project $PROJECT -instance $INSTANCE createfamily beaconchain_validator_sync sc +cbt -project $PROJECT -instance $INSTANCE createfamily beaconchain_validator_sync sp ``` ---- Table name: `beaconchain_validator_income` From 79b87531c9cc8ae8587e82eb60b69df5a61e27f9 Mon Sep 17 00:00:00 2001 From: peter <1674920+peterbitfly@users.noreply.github.com> Date: Fri, 1 Sep 2023 10:53:06 +0200 Subject: [PATCH 066/106] (BIDS-2379) handle income for new chains correctly, remove obsole bigtable cache module --- cache/bigtable_cache.go | 204 ------------------------------ cache/tiered_cache.go | 13 -- cmd/explorer/main.go | 6 +- cmd/frontend-data-updater/main.go | 4 +- db/bigtable.go | 114 ++++++++++------- db/bigtable_admin.go | 44 ------- db/db.go | 25 +++- db/statistics.go | 24 +++- exporter/sync_committees.go | 15 ++- handlers/api.go | 7 +- handlers/api_eth1.go | 7 +- handlers/common.go | 41 ++++-- handlers/dashboard.go | 4 +- handlers/validator.go | 2 +- local-deployment/main.star | 6 +- services/notifications.go | 7 +- services/services.go | 16 ++- 17 files changed, 178 insertions(+), 361 deletions(-) delete mode 100644 cache/bigtable_cache.go diff --git a/cache/bigtable_cache.go b/cache/bigtable_cache.go deleted file mode 100644 index 7f45d56320..0000000000 --- a/cache/bigtable_cache.go +++ /dev/null @@ -1,204 +0,0 @@ -package cache - -import ( - "context" - "encoding/json" - "fmt" - "time" - - gcp_bigtable "cloud.google.com/go/bigtable" - "github.com/sirupsen/logrus" -) - -const ( - TABLE_CACHE = "cache" - FAMILY_TEN_MINUTES = "10_min" - FAMILY_ONE_HOUR = "1_hour" - FAMILY_ONE_DAY = "1_day" - COLUMN_DATA = "d" -) - -type BigtableCache struct { - client *gcp_bigtable.Client - - tableCache *gcp_bigtable.Table - - chainId string -} - -func InitBigtableCache(client *gcp_bigtable.Client, chainId string) *BigtableCache { - bt := &BigtableCache{ - client: client, - tableCache: client.Open(TABLE_CACHE), - chainId: chainId, - } - - return bt -} - -func (cache *BigtableCache) Set(ctx context.Context, key string, value any, expiration time.Duration) error { - - family := FAMILY_TEN_MINUTES - if expiration.Minutes() >= 60 { - family = FAMILY_ONE_HOUR - } - if expiration.Hours() > 1 { - family = FAMILY_ONE_DAY - } - - valueMarshal, err := json.Marshal(value) - if err != nil { - return err - } - - ts := gcp_bigtable.Now() - mut := gcp_bigtable.NewMutation() - mut.Set(family, COLUMN_DATA, ts, valueMarshal) - - err = cache.tableCache.Apply(ctx, fmt.Sprintf("C:%s", key), mut) - if err != nil { - return err - } - return nil -} - -func (cache *BigtableCache) setByte(ctx context.Context, key string, value []byte, expiration time.Duration) error { - - family := FAMILY_TEN_MINUTES - if expiration.Minutes() >= 60 { - family = FAMILY_ONE_HOUR - } - if expiration.Hours() > 1 { - family = FAMILY_ONE_DAY - } - - ts := gcp_bigtable.Now() - mut := gcp_bigtable.NewMutation() - mut.Set(family, COLUMN_DATA, ts, value) - - err := cache.tableCache.Apply(ctx, fmt.Sprintf("C:%s", key), mut) - if err != nil { - return err - } - return nil -} - -func (cache *BigtableCache) SetString(ctx context.Context, key, value string, expiration time.Duration) error { - return cache.setByte(ctx, key, []byte(value), expiration) -} - -func (cache *BigtableCache) SetUint64(ctx context.Context, key string, value uint64, expiration time.Duration) error { - return cache.setByte(ctx, key, ui64tob(value), expiration) -} - -func (cache *BigtableCache) SetBool(ctx context.Context, key string, value bool, expiration time.Duration) error { - return cache.setByte(ctx, key, booltob(value), expiration) -} - -func (cache *BigtableCache) Get(ctx context.Context, key string, returnValue any) (any, error) { - res, err := cache.getByte(ctx, key) - if err != nil { - return nil, err - } - - err = json.Unmarshal([]byte(res), returnValue) - if err != nil { - // cache.remoteRedisCache.Del(ctx, key).Err() - logrus.Errorf("error (bigtable_cache.go / Get) unmarshalling data for key %v: %v", key, err) - return nil, err - } - - return returnValue, nil -} - -func (cache *BigtableCache) getByte(ctx context.Context, key string) ([]byte, error) { - filter := gcp_bigtable.ChainFilters( - gcp_bigtable.ColumnFilter("d"), - gcp_bigtable.LatestNFilter(1), - ) - - row, err := cache.tableCache.ReadRow(ctx, fmt.Sprintf("C:%s", key), gcp_bigtable.RowFilter(filter)) - if err != nil { - return nil, err - } - - if len(row) == 0 { - return nil, fmt.Errorf("error getting key: %s no result available, row: %+v", key, row) - } - - // iterate over all column families and only take the most recent entry - res := new(gcp_bigtable.ReadItem) - for _, column := range row { - if len(column) != 1 { - return nil, fmt.Errorf("error unexpected number of results returned key: %s no result available, row: %+v", key, row) - } - if res == nil { - res = &column[0] - } - - if res.Timestamp.Time().Before(column[0].Timestamp.Time()) { - res = &column[0] - } - } - - return res.Value, nil -} - -func (cache *BigtableCache) GetString(ctx context.Context, key string) (string, error) { - - res, err := cache.getByte(ctx, key) - if err != nil { - return "", err - } - return string(res), nil -} - -func (cache *BigtableCache) GetUint64(ctx context.Context, key string) (uint64, error) { - - res, err := cache.getByte(ctx, key) - if err != nil { - return 0, err - } - - return btoi64(res), nil -} - -func (cache *BigtableCache) GetBool(ctx context.Context, key string) (bool, error) { - - res, err := cache.getByte(ctx, key) - if err != nil { - return false, err - } - - return btobool(res), nil -} - -func ui64tob(val uint64) []byte { - r := make([]byte, 8) - for i := uint64(0); i < 8; i++ { - r[i] = byte((val >> (i * 8)) & 0xff) - } - return r -} - -func btoi64(val []byte) uint64 { - r := uint64(0) - for i := uint64(0); i < 8; i++ { - r |= uint64(val[i]) << (8 * i) - } - return r -} - -func booltob(val bool) []byte { - r := make([]byte, 1) - if val { - r[0] = 1 - } else { - r[0] = 0 - } - return r -} - -func btobool(val []byte) bool { - return val[0] == 1 -} diff --git a/cache/tiered_cache.go b/cache/tiered_cache.go index 007cecb650..531c7bf837 100644 --- a/cache/tiered_cache.go +++ b/cache/tiered_cache.go @@ -8,7 +8,6 @@ import ( "strconv" "time" - gcp_bigtable "cloud.google.com/go/bigtable" "github.com/coocood/freecache" "github.com/sirupsen/logrus" ) @@ -48,18 +47,6 @@ func MustInitTieredCache(redisAddress string) { } } -func MustInitTieredCacheBigtable(client *gcp_bigtable.Client, chainId string) { - localCache := freecache.NewCache(100 * 1024 * 1024) // 100 MB - - cache := InitBigtableCache(client, chainId) - - TieredCache = &tieredCache{ - remoteCache: cache, - localGoCache: localCache, - } - -} - func (cache *tieredCache) SetString(key, value string, expiration time.Duration) error { ctx, cancel := context.WithTimeout(context.Background(), time.Second*30) defer cancel() diff --git a/cmd/explorer/main.go b/cmd/explorer/main.go index ea12f98462..a0975be184 100644 --- a/cmd/explorer/main.go +++ b/cmd/explorer/main.go @@ -190,12 +190,8 @@ func main() { } wg.Wait() - if utils.Config.TieredCacheProvider == "bigtable" && len(utils.Config.RedisCacheEndpoint) == 0 { - cache.MustInitTieredCacheBigtable(db.BigtableClient.GetClient(), fmt.Sprintf("%d", utils.Config.Chain.Config.DepositChainID)) - logrus.Infof("tiered Cache initialized, latest finalized epoch: %v", services.LatestFinalizedEpoch()) - } - if utils.Config.TieredCacheProvider != "bigtable" && utils.Config.TieredCacheProvider != "redis" { + if utils.Config.TieredCacheProvider != "redis" { logrus.Fatalf("no cache provider set, please set TierdCacheProvider (example redis, bigtable)") } diff --git a/cmd/frontend-data-updater/main.go b/cmd/frontend-data-updater/main.go index c87f85bb69..5955c3392f 100644 --- a/cmd/frontend-data-updater/main.go +++ b/cmd/frontend-data-updater/main.go @@ -70,11 +70,9 @@ func main() { if utils.Config.TieredCacheProvider == "redis" || len(utils.Config.RedisCacheEndpoint) != 0 { cache.MustInitTieredCache(utils.Config.RedisCacheEndpoint) - } else if utils.Config.TieredCacheProvider == "bigtable" && len(utils.Config.RedisCacheEndpoint) == 0 { - cache.MustInitTieredCacheBigtable(db.BigtableClient.GetClient(), fmt.Sprintf("%d", utils.Config.Chain.Config.DepositChainID)) } - if utils.Config.TieredCacheProvider != "bigtable" && utils.Config.TieredCacheProvider != "redis" { + if utils.Config.TieredCacheProvider != "redis" { logrus.Fatalf("No cache provider set. Please set TierdCacheProvider (example redis, bigtable)") } diff --git a/db/bigtable.go b/db/bigtable.go index e349931491..a5e3fe5ef9 100644 --- a/db/bigtable.go +++ b/db/bigtable.go @@ -591,65 +591,63 @@ func (bigtable *Bigtable) SaveProposalAssignments(epoch uint64, assignments map[ func (bigtable *Bigtable) SaveSyncCommitteesAssignments(startSlot, endSlot uint64, validators []uint64) error { - return nil //disabled as not needed - - // ctx, cancel := context.WithTimeout(context.Background(), time.Minute*10) - // defer cancel() + ctx, cancel := context.WithTimeout(context.Background(), time.Minute*10) + defer cancel() - // start := time.Now() - // ts := gcp_bigtable.Timestamp(0) + start := time.Now() + ts := gcp_bigtable.Timestamp(0) - // muts := make([]*gcp_bigtable.Mutation, 0, MAX_BATCH_MUTATIONS) - // keys := make([]string, 0, MAX_BATCH_MUTATIONS) + muts := make([]*gcp_bigtable.Mutation, 0, MAX_BATCH_MUTATIONS) + keys := make([]string, 0, MAX_BATCH_MUTATIONS) - // for i := startSlot; i <= endSlot; i++ { - // for _, validator := range validators { - // mut := gcp_bigtable.NewMutation() - // mut.Set(SYNC_COMMITTEES_FAMILY, "s", ts, []byte{}) + for i := startSlot; i <= endSlot; i++ { + for _, validator := range validators { + mut := gcp_bigtable.NewMutation() + mut.Set(SYNC_COMMITTEES_FAMILY, "s", ts, []byte{}) - // key := fmt.Sprintf("%s:%s:%s:%s", bigtable.chainId, bigtable.validatorIndexToKey(validator), bigtable.reversedPaddedEpoch(utils.EpochOfSlot(i)), bigtable.reversedPaddedSlot(i)) + key := fmt.Sprintf("%s:%s:%s:%s", bigtable.chainId, bigtable.validatorIndexToKey(validator), bigtable.reversedPaddedEpoch(utils.EpochOfSlot(i)), bigtable.reversedPaddedSlot(i)) - // muts = append(muts, mut) - // keys = append(keys, key) + muts = append(muts, mut) + keys = append(keys, key) - // if len(muts) == MAX_BATCH_MUTATIONS { - // logger.Infof("saving %v mutations for sync duties", len(muts)) - // errs, err := bigtable.tableValidatorSyncCommittees.ApplyBulk(ctx, keys, muts) + if len(muts) == MAX_BATCH_MUTATIONS { + logger.Infof("saving %v mutations for sync duties", len(muts)) + errs, err := bigtable.tableValidatorSyncCommittees.ApplyBulk(ctx, keys, muts) - // if err != nil { - // return err - // } + if err != nil { + return err + } - // for _, err := range errs { - // if err != nil { - // return err - // } - // } + for _, err := range errs { + if err != nil { + return err + } + } - // muts = make([]*gcp_bigtable.Mutation, 0, MAX_BATCH_MUTATIONS) - // keys = make([]string, 0, MAX_BATCH_MUTATIONS) - // } - // } + muts = make([]*gcp_bigtable.Mutation, 0, MAX_BATCH_MUTATIONS) + keys = make([]string, 0, MAX_BATCH_MUTATIONS) + } + } - // } + } - // if len(muts) > 0 { - // logger.Infof("saving %v mutations for sync duties", len(muts)) - // errs, err := bigtable.tableValidatorSyncCommittees.ApplyBulk(ctx, keys, muts) + if len(muts) > 0 { + logger.Infof("saving %v mutations for sync duties", len(muts)) + errs, err := bigtable.tableValidatorSyncCommittees.ApplyBulk(ctx, keys, muts) - // if err != nil { - // return err - // } + if err != nil { + return err + } - // for _, err := range errs { - // if err != nil { - // return err - // } - // } - // } + for _, err := range errs { + if err != nil { + return err + } + } + } - // logger.Infof("exported sync committee assignments to bigtable in %v", time.Since(start)) - // return nil + logger.Infof("exported sync committee assignments to bigtable in %v", time.Since(start)) + return nil } func (bigtable *Bigtable) SaveAttestations(blocks map[uint64]map[string]*types.Block) error { @@ -884,6 +882,7 @@ func (bigtable *Bigtable) SaveSyncComitteeDuties(blocks map[uint64]map[string]*t mut.Set(SYNC_COMMITTEES_FAMILY, "s", gcp_bigtable.Timestamp(0), []byte{}) } key := fmt.Sprintf("%s:%s:%s:%s", bigtable.chainId, bigtable.validatorIndexToKey(validator), bigtable.reversedPaddedEpoch(utils.EpochOfSlot(slot)), bigtable.reversedPaddedSlot(slot)) + muts = append(muts, mut) keys = append(keys, key) } @@ -1347,7 +1346,8 @@ func (bigtable *Bigtable) GetValidatorSyncDutiesHistory(validators []uint64, sta return gCtx.Err() default: } - ranges := bigtable.getValidatorsEpochRanges(vals, startEpoch, endEpoch) + ranges := bigtable.getValidatorsEpochSlotRanges(vals, startEpoch, endEpoch) + err := bigtable.tableValidatorSyncCommittees.ReadRows(ctx, ranges, func(r gcp_bigtable.Row) bool { for _, ri := range r[SYNC_COMMITTEES_FAMILY] { @@ -1632,7 +1632,7 @@ func (bigtable *Bigtable) GetValidatorProposalHistory(validators []uint64, start return gCtx.Err() default: } - ranges := bigtable.getValidatorsEpochRanges(vals, startEpoch, endEpoch) + ranges := bigtable.getValidatorsEpochSlotRanges(vals, startEpoch, endEpoch) err := bigtable.tableValidatorProposals.ReadRows(ctx, ranges, func(r gcp_bigtable.Row) bool { for _, ri := range r[PROPOSALS_FAMILY] { keySplit := strings.Split(r.Key(), ":") @@ -2260,7 +2260,25 @@ func (bigtable *Bigtable) getValidatorsEpochRanges(validatorIndices []uint64, st // epochs are sorted descending, so start with the largest epoch and end with the smallest // add \x00 to make the range inclusive rangeEnd := fmt.Sprintf("%s:%s:%s%s", bigtable.chainId, validatorKey, bigtable.reversedPaddedEpoch(startEpoch), "\x00") - rangeStart := fmt.Sprintf("%s:%s:%s", bigtable.chainId, bigtable.validatorIndexToKey(validatorIndex), bigtable.reversedPaddedEpoch(endEpoch)) + rangeStart := fmt.Sprintf("%s:%s:%s", bigtable.chainId, validatorKey, bigtable.reversedPaddedEpoch(endEpoch)) + ranges = append(ranges, gcp_bigtable.NewRange(rangeStart, rangeEnd)) + } + return ranges +} + +func (bigtable *Bigtable) getValidatorsEpochSlotRanges(validatorIndices []uint64, startEpoch uint64, endEpoch uint64) gcp_bigtable.RowRangeList { + + if endEpoch < startEpoch { // handle overflows + startEpoch = 0 + } + + ranges := make(gcp_bigtable.RowRangeList, 0, int((endEpoch-startEpoch))*len(validatorIndices)) + + for _, validatorIndex := range validatorIndices { + validatorKey := bigtable.validatorIndexToKey(validatorIndex) + + rangeEnd := fmt.Sprintf("%s:%s:%s:%s%s", bigtable.chainId, validatorKey, bigtable.reversedPaddedEpoch(startEpoch), bigtable.reversedPaddedSlot(startEpoch*utils.Config.Chain.Config.SlotsPerEpoch), "\x00") + rangeStart := fmt.Sprintf("%s:%s:%s:%s", bigtable.chainId, validatorKey, bigtable.reversedPaddedEpoch(endEpoch), bigtable.reversedPaddedSlot(endEpoch*utils.Config.Chain.Config.SlotsPerEpoch+utils.Config.Chain.Config.SlotsPerEpoch-1)) ranges = append(ranges, gcp_bigtable.NewRange(rangeStart, rangeEnd)) } diff --git a/db/bigtable_admin.go b/db/bigtable_admin.go index 338139f331..20e48feb55 100644 --- a/db/bigtable_admin.go +++ b/db/bigtable_admin.go @@ -2,10 +2,8 @@ package db import ( "context" - "eth2-exporter/cache" "eth2-exporter/utils" "log" - "time" gcp_bigtable "cloud.google.com/go/bigtable" ) @@ -24,24 +22,6 @@ type CreateFamily struct { Policy gcp_bigtable.GCPolicy } -var CacheTable CreateTables = CreateTables{ - cache.TABLE_CACHE, - []CreateFamily{ - { - Name: cache.FAMILY_TEN_MINUTES, - Policy: gcp_bigtable.IntersectionPolicy(gcp_bigtable.MaxVersionsPolicy(1), gcp_bigtable.MaxAgePolicy(time.Minute*10)), - }, - { - Name: cache.FAMILY_ONE_HOUR, - Policy: gcp_bigtable.IntersectionPolicy(gcp_bigtable.MaxVersionsPolicy(1), gcp_bigtable.MaxAgePolicy(time.Hour)), - }, - { - Name: cache.FAMILY_ONE_DAY, - Policy: gcp_bigtable.IntersectionPolicy(gcp_bigtable.MaxVersionsPolicy(1), gcp_bigtable.MaxAgePolicy(time.Hour*24)), - }, - }, -} - var BigAdminClient *BigtableAdmin func MustInitBigtableAdmin(ctx context.Context, project, instance string) { @@ -57,30 +37,6 @@ func MustInitBigtableAdmin(ctx context.Context, project, instance string) { BigAdminClient = bta } -func (admin *BigtableAdmin) SetupBigtableCache() error { - - if err := admin.createTables([]CreateTables{CacheTable}); err != nil { - log.Fatal("Error occurred trying to create tables", err) - } - ctx, done := context.WithTimeout(context.Background(), time.Second*30) - defer done() - - for _, cf := range CacheTable.ColFams { - if err := admin.client.SetGCPolicy(ctx, CacheTable.Name, cf.Name, cf.Policy); err != nil { - return err - } - } - - return nil -} - -func (admin *BigtableAdmin) TearDownCache() error { - if err := admin.deleteTables([]CreateTables{CacheTable}); err != nil { - return err - } - return nil -} - func (admin *BigtableAdmin) createTables(tables []CreateTables) error { ctx := context.Background() diff --git a/db/db.go b/db/db.go index 053fbc8fff..d0575f1260 100644 --- a/db/db.go +++ b/db/db.go @@ -6,6 +6,7 @@ import ( "database/sql" "embed" "encoding/hex" + "errors" "eth2-exporter/metrics" "eth2-exporter/types" "eth2-exporter/utils" @@ -54,6 +55,8 @@ const MaxSqlInteger = 2147483647 var addressRE = regexp.MustCompile(`^(0x)?[0-9a-fA-F]{40}$`) var blsRE = regexp.MustCompile(`^(0x)?[0-9a-fA-F]{96}$`) +var ErrNoStats = errors.New("no stats available") + func dbTestConnection(dbConn *sqlx.DB, dataBaseName string) { // The golang sql driver does not properly implement PingContext // therefore we use a timer to catch db connection timeouts @@ -3209,16 +3212,20 @@ func GetPendingBLSChangeValidatorCount() (uint64, error) { } func GetLastExportedStatisticDay() (uint64, error) { - var lastStatsDay uint64 - err := ReaderDb.Get(&lastStatsDay, "SELECT COALESCE(MAX(day),0) FROM validator_stats_status WHERE status") + var lastStatsDay sql.NullInt64 + err := ReaderDb.Get(&lastStatsDay, "SELECT MAX(day) FROM validator_stats_status WHERE status") if err != nil { return 0, fmt.Errorf("error getting lastStatsDay %v", err) } - return lastStatsDay, nil + + if !lastStatsDay.Valid { + return 0, ErrNoStats + } + return uint64(lastStatsDay.Int64), nil } -func GetValidatorIncomePerforamance(validators []uint64, incomePerformance *types.ValidatorIncomePerformance) error { +func GetValidatorIncomePerformance(validators []uint64, incomePerformance *types.ValidatorIncomePerformance) error { validatorsPQArray := pq.Array(validators) // el rewards are converted from wei to gwei return ReaderDb.Get(incomePerformance, ` @@ -3301,6 +3308,16 @@ func GetValidatorBalanceForDay(validators []uint64, day uint64, balance *uint64) `, validatorsPQArray, day) } +func GetValidatorActivationBalance(validators []uint64, balance *uint64) error { + validatorsPQArray := pq.Array(validators) + return ReaderDb.Get(balance, ` + SELECT + SUM(balanceactivation) + FROM validators + WHERE validatorindex = ANY($1) + `, validatorsPQArray) +} + func GetValidatorPropsosals(validators []uint64, proposals *[]types.ValidatorProposalInfo) error { validatorsPQArray := pq.Array(validators) diff --git a/db/statistics.go b/db/statistics.go index 2fad68c988..f9055a4b42 100644 --- a/db/statistics.go +++ b/db/statistics.go @@ -1261,18 +1261,25 @@ func GetValidatorIncomeHistory(validatorIndices []uint64, lowerBoundDay uint64, // retrieve rewards for epochs not yet in stats if upperBoundDay == 65536 { - lastDay := uint64(0) + lastDay := int64(0) if len(result) > 0 { - lastDay = uint64(result[len(result)-1].Day) + lastDay = int64(result[len(result)-1].Day) } else { - lastDay, err = GetLastExportedStatisticDay() - if err != nil { + lastDayDb, err := GetLastExportedStatisticDay() + if err == nil { + lastDay = int64(lastDayDb) + } else if err == ErrNoStats { + lastDay = -1 + } else { return nil, err } } currentDay := lastDay + 1 - firstSlot := utils.GetLastBalanceInfoSlotForDay(lastDay) + 1 + firstSlot := uint64(0) + if lastDay > -1 { + firstSlot = utils.GetLastBalanceInfoSlotForDay(uint64(lastDay)) + 1 + } lastSlot := lastFinalizedEpoch * utils.Config.Chain.Config.SlotsPerEpoch totalBalance := uint64(0) @@ -1297,7 +1304,12 @@ func GetValidatorIncomeHistory(validatorIndices []uint64, lowerBoundDay uint64, var lastBalance uint64 g.Go(func() error { - return GetValidatorBalanceForDay(validatorIndices, lastDay, &lastBalance) + + if lastDay < 0 { + return GetValidatorActivationBalance(validatorIndices, &lastBalance) + } else { + return GetValidatorBalanceForDay(validatorIndices, uint64(lastDay), &lastBalance) + } }) var lastDeposits uint64 diff --git a/exporter/sync_committees.go b/exporter/sync_committees.go index 55a0f48b2d..258507d73d 100644 --- a/exporter/sync_committees.go +++ b/exporter/sync_committees.go @@ -86,6 +86,17 @@ func exportSyncCommitteeAtPeriod(rpcClient rpc.Client, p uint64) error { validatorsU64[i] = idxU64 } + dedupMap := make(map[uint64]bool, len(validatorsU64)) + + for _, validator := range validatorsU64 { + dedupMap[validator] = true + } + + validatorsU64 = make([]uint64, len(dedupMap)) + for validator := range dedupMap { + validatorsU64 = append(validatorsU64, validator) + } + start := time.Now() firstSlot := firstEpoch * utils.Config.Chain.Config.SlotsPerEpoch lastSlot := lastEpoch*utils.Config.Chain.Config.SlotsPerEpoch + utils.Config.Chain.Config.SlotsPerEpoch - 1 @@ -104,8 +115,8 @@ func exportSyncCommitteeAtPeriod(rpcClient rpc.Client, p uint64) error { defer tx.Rollback() nArgs := 3 - valueArgs := make([]interface{}, len(c.Validators)*nArgs) - valueIds := make([]string, len(c.Validators)) + valueArgs := make([]interface{}, len(validatorsU64)*nArgs) + valueIds := make([]string, len(validatorsU64)) for i, idxU64 := range validatorsU64 { valueArgs[i*nArgs+0] = p valueArgs[i*nArgs+1] = idxU64 diff --git a/handlers/api.go b/handlers/api.go index eef5a7eec0..761444f785 100644 --- a/handlers/api.go +++ b/handlers/api.go @@ -1177,9 +1177,12 @@ func getSyncCommitteeSlotsStatistics(validators []uint64, epoch uint64) (types.S // validator_stats is updated only once a day, everything missing has to be collected from bigtable (which is slower than validator_stats) // check when the last update to validator_stats was - lastExportedDay := services.LatestExportedStatisticDay() epochsPerDay := utils.EpochsPerDay() - lastExportedEpoch := ((lastExportedDay + 1) * epochsPerDay) - 1 + lastExportedDay, err := services.LatestExportedStatisticDay() + lastExportedEpoch := uint64(0) + if err == nil { + lastExportedEpoch = ((lastExportedDay + 1) * epochsPerDay) - 1 + } // if epoch is not yet exported, we may need to collect the data from bigtable if lastExportedEpoch < epoch { diff --git a/handlers/api_eth1.go b/handlers/api_eth1.go index 5dfd16f6eb..6c6e42193f 100644 --- a/handlers/api_eth1.go +++ b/handlers/api_eth1.go @@ -959,8 +959,11 @@ func getValidatorExecutionPerformance(queryIndices []uint64) ([]types.ExecutionP } } - lastStatsDay := services.LatestExportedStatisticDay() - firstEpochTime := utils.EpochToTime((lastStatsDay + 1) * utils.EpochsPerDay()) + lastStatsDay, err := services.LatestExportedStatisticDay() + firstEpochTime := utils.EpochToTime(0) + if err == nil { + firstEpochTime = utils.EpochToTime((lastStatsDay + 1) * utils.EpochsPerDay()) + } for _, block := range blocks { proposer := blockToProposerMap[block.Number].Proposer diff --git a/handlers/common.go b/handlers/common.go index 05aa819714..0ab69a0a29 100644 --- a/handlers/common.go +++ b/handlers/common.go @@ -47,8 +47,13 @@ func GetValidatorEarnings(validators []uint64, currency string) (*types.Validato return nil, nil, errors.New("no validators provided") } latestFinalizedEpoch := services.LatestFinalizedEpoch() - lastStatsDay := services.LatestExportedStatisticDay() - firstSlot := utils.GetLastBalanceInfoSlotForDay(lastStatsDay) + 1 + + firstSlot := uint64(0) + lastStatsDay, lastExportedStatsErr := services.LatestExportedStatisticDay() + if lastExportedStatsErr == nil { + firstSlot = utils.GetLastBalanceInfoSlotForDay(lastStatsDay) + 1 + } + lastSlot := latestFinalizedEpoch * utils.Config.Chain.Config.SlotsPerEpoch balancesMap := make(map[uint64]*types.Validator, 0) @@ -80,7 +85,7 @@ func GetValidatorEarnings(validators []uint64, currency string) (*types.Validato income := types.ValidatorIncomePerformance{} g.Go(func() error { - return db.GetValidatorIncomePerforamance(validators, &income) + return db.GetValidatorIncomePerformance(validators, &income) }) var totalDeposits uint64 @@ -99,18 +104,25 @@ func GetValidatorEarnings(validators []uint64, currency string) (*types.Validato }) var lastDeposits uint64 - g.Go(func() error { - return db.GetValidatorDepositsForSlots(validators, firstSlot, lastSlot, &lastDeposits) - }) - var lastWithdrawals uint64 - g.Go(func() error { - return db.GetValidatorWithdrawalsForSlots(validators, firstSlot, lastSlot, &lastWithdrawals) - }) - var lastBalance uint64 g.Go(func() error { - return db.GetValidatorBalanceForDay(validators, lastStatsDay, &lastBalance) + if lastExportedStatsErr == db.ErrNoStats { + err := db.GetValidatorActivationBalance(validators, &lastBalance) + if err != nil { + return err + } + } else { + err := db.GetValidatorBalanceForDay(validators, lastStatsDay, &lastBalance) + if err != nil { + return err + } + } + err := db.GetValidatorDepositsForSlots(validators, firstSlot, lastSlot, &lastDeposits) + if err != nil { + return err + } + return db.GetValidatorWithdrawalsForSlots(validators, firstSlot, lastSlot, &lastWithdrawals) }) proposals := []types.ValidatorProposalInfo{} @@ -169,7 +181,10 @@ func GetValidatorEarnings(validators []uint64, currency string) (*types.Validato } proposedToday := []uint64{} - todayStartEpoch := uint64(lastStatsDay+1) * utils.EpochsPerDay() + todayStartEpoch := uint64(0) + if lastExportedStatsErr == nil { + todayStartEpoch = uint64(lastStatsDay+1) * utils.EpochsPerDay() + } validatorProposalData := types.ValidatorProposalData{} validatorProposalData.Proposals = make([][]uint64, len(proposals)) for i, b := range proposals { diff --git a/handlers/dashboard.go b/handlers/dashboard.go index 51729652a9..68e6c49706 100644 --- a/handlers/dashboard.go +++ b/handlers/dashboard.go @@ -454,8 +454,8 @@ func DashboardDataBalanceCombined(w http.ResponseWriter, r *http.Request) { http.Error(w, "Error: invalid days parameter", http.StatusBadRequest) return } - lastStatsDay := services.LatestExportedStatisticDay() - if days < lastStatsDay { + lastStatsDay, err := services.LatestExportedStatisticDay() + if days < lastStatsDay && err == nil { lowerBoundDay = lastStatsDay - days + 1 } } diff --git a/handlers/validator.go b/handlers/validator.go index 0f003fe9ea..b48b821cc0 100644 --- a/handlers/validator.go +++ b/handlers/validator.go @@ -299,7 +299,7 @@ func Validator(w http.ResponseWriter, r *http.Request) { } validatorPageData.LastAttestationSlot = lastAttestationSlots[index] - lastStatsDay := services.LatestExportedStatisticDay() + lastStatsDay, _ := services.LatestExportedStatisticDay() timings.BasicInfo = time.Since(timings.Start) diff --git a/local-deployment/main.star b/local-deployment/main.star index 396d77bdf0..e33e7cf18a 100644 --- a/local-deployment/main.star +++ b/local-deployment/main.star @@ -57,7 +57,7 @@ def run(plan, args): all_el_client_contexts.append(participant.el_client_context) all_cl_client_contexts.append(participant.cl_client_context) - plan.print("Launching transaction spammer") - transaction_spammer.launch_transaction_spammer(plan, genesis_constants.PRE_FUNDED_ACCOUNTS, all_el_client_contexts[0]) - plan.print("Succesfully launched transaction spammer") + # plan.print("Launching transaction spammer") + # transaction_spammer.launch_transaction_spammer(plan, genesis_constants.PRE_FUNDED_ACCOUNTS, all_el_client_contexts[0]) + # plan.print("Succesfully launched transaction spammer") diff --git a/services/notifications.go b/services/notifications.go index 183e72e827..1e213f3f73 100644 --- a/services/notifications.go +++ b/services/notifications.go @@ -2593,8 +2593,11 @@ func (n *taxReportNotification) GetInfoMarkdown() string { } func collectTaxReportNotificationNotifications(notificationsByUserID map[uint64]map[types.EventName][]types.Notification, eventName types.EventName) error { - lastStatsDay := LatestExportedStatisticDay() + lastStatsDay, err := LatestExportedStatisticDay() + if err != nil { + return err + } //Check that the last day of the month is already exported tNow := time.Now() firstDayOfMonth := time.Date(tNow.Year(), tNow.Month(), 1, 0, 0, 0, 0, time.UTC) @@ -2615,7 +2618,7 @@ func collectTaxReportNotificationNotifications(notificationsByUserID map[uint64] name = utils.Config.Chain.Config.ConfigName + ":" + name } - err := db.FrontendWriterDB.Select(&dbResult, ` + err = db.FrontendWriterDB.Select(&dbResult, ` SELECT us.id, us.user_id, us.created_epoch, us.event_filter, ENCODE(us.unsubscribe_hash, 'hex') as unsubscribe_hash FROM users_subscriptions AS us WHERE us.event_name=$1 AND (us.last_sent_ts < $2 OR (us.last_sent_ts IS NULL AND us.created_ts < $2)); diff --git a/services/services.go b/services/services.go index 6466703e59..aca5d543a3 100644 --- a/services/services.go +++ b/services/services.go @@ -1674,6 +1674,7 @@ func getBurnPageData() (*types.BurnPageData, error) { func latestExportedStatisticDayUpdater(wg *sync.WaitGroup) { firstRun := true + cacheKey := fmt.Sprintf("%d:frontend:lastExportedStatisticDay", utils.Config.Chain.Config.DepositChainID) for { lastDay, err := db.GetLastExportedStatisticDay() if err != nil { @@ -1682,7 +1683,6 @@ func latestExportedStatisticDayUpdater(wg *sync.WaitGroup) { continue } - cacheKey := fmt.Sprintf("%d:frontend:lastExportedStatisticDay", utils.Config.Chain.Config.DepositChainID) err = cache.TieredCache.Set(cacheKey, lastDay, time.Hour*24) if err != nil { logger.Errorf("error caching last exported statistics day: %v", err) @@ -1698,14 +1698,16 @@ func latestExportedStatisticDayUpdater(wg *sync.WaitGroup) { } // LatestExportedStatisticDay will return the last exported day in the validator_stats table -func LatestExportedStatisticDay() uint64 { +func LatestExportedStatisticDay() (uint64, error) { cacheKey := fmt.Sprintf("%d:frontend:lastExportedStatisticDay", utils.Config.Chain.Config.DepositChainID) if wanted, err := cache.TieredCache.GetUint64WithLocalTimeout(cacheKey, time.Second*5); err == nil { - return wanted - } else { - logger.Errorf("error retrieving last exported statistics day from cache: %v", err) + return wanted, nil + } + wanted, err := db.GetLastExportedStatisticDay() + + if err != nil { + return 0, err } - wanted, _ := db.GetLastExportedStatisticDay() - return wanted + return wanted, nil } From 62d32b4d043ca1a5f11263bfde823be64c659b1b Mon Sep 17 00:00:00 2001 From: peter <1674920+peterbitfly@users.noreply.github.com> Date: Fri, 1 Sep 2023 10:54:56 +0200 Subject: [PATCH 067/106] (BIDS-2379) fix staticcheck errors --- cmd/statistics/main.go | 10 ++--- db/bigtable_admin.go | 86 ------------------------------------------ 2 files changed, 4 insertions(+), 92 deletions(-) delete mode 100644 db/bigtable_admin.go diff --git a/cmd/statistics/main.go b/cmd/statistics/main.go index 8243d68aed..bcbf2f710a 100644 --- a/cmd/statistics/main.go +++ b/cmd/statistics/main.go @@ -116,14 +116,12 @@ func main() { price.Init(utils.Config.Chain.Config.DepositChainID, utils.Config.Eth1ErigonEndpoint) - if utils.Config.TieredCacheProvider == "redis" || len(utils.Config.RedisCacheEndpoint) != 0 { - cache.MustInitTieredCache(utils.Config.RedisCacheEndpoint) - } else if utils.Config.TieredCacheProvider == "bigtable" && len(utils.Config.RedisCacheEndpoint) == 0 { - cache.MustInitTieredCacheBigtable(db.BigtableClient.GetClient(), fmt.Sprintf("%d", utils.Config.Chain.Config.DepositChainID)) + if utils.Config.TieredCacheProvider != "redis" { + logrus.Fatalf("No cache provider set. Please set TierdCacheProvider (example redis)") } - if utils.Config.TieredCacheProvider != "bigtable" && utils.Config.TieredCacheProvider != "redis" { - logrus.Fatalf("No cache provider set. Please set TierdCacheProvider (example redis, bigtable)") + if utils.Config.TieredCacheProvider == "redis" || len(utils.Config.RedisCacheEndpoint) != 0 { + cache.MustInitTieredCache(utils.Config.RedisCacheEndpoint) } if opt.statisticsDaysToExport != "" { diff --git a/db/bigtable_admin.go b/db/bigtable_admin.go deleted file mode 100644 index 20e48feb55..0000000000 --- a/db/bigtable_admin.go +++ /dev/null @@ -1,86 +0,0 @@ -package db - -import ( - "context" - "eth2-exporter/utils" - "log" - - gcp_bigtable "cloud.google.com/go/bigtable" -) - -type BigtableAdmin struct { - client *gcp_bigtable.AdminClient -} - -type CreateTables struct { - Name string - ColFams []CreateFamily -} - -type CreateFamily struct { - Name string - Policy gcp_bigtable.GCPolicy -} - -var BigAdminClient *BigtableAdmin - -func MustInitBigtableAdmin(ctx context.Context, project, instance string) { - admin, err := gcp_bigtable.NewAdminClient(ctx, project, instance) - if err != nil { - log.Fatalf("Could not create admin client: %v", err) - } - - bta := &BigtableAdmin{ - client: admin, - } - - BigAdminClient = bta -} - -func (admin *BigtableAdmin) createTables(tables []CreateTables) error { - ctx := context.Background() - - tableList, err := admin.client.Tables(ctx) - if err != nil { - log.Printf("Could not fetch table list") - return err - } - - for _, table := range tables { - if !utils.SliceContains(tableList, table.Name) { - log.Printf("Creating table %s", table) - if err := admin.client.CreateTable(ctx, table.Name); err != nil { - log.Printf("Could not create table %s", table.Name) - return err - } - } - - tblInfo, err := admin.client.TableInfo(ctx, table.Name) - if err != nil { - log.Printf("Could not read info for table %s", table.Name) - return err - } - for _, colfam := range table.ColFams { - if !utils.SliceContains(tblInfo.Families, colfam.Name) { - if err := admin.client.CreateColumnFamily(ctx, table.Name, colfam.Name); err != nil { - log.Printf("Could not create column family %s: %v", colfam.Name, err) - return err - } - } - } - } - return nil -} - -func (admin *BigtableAdmin) deleteTables(tables []CreateTables) error { - ctx := context.Background() - for _, table := range tables { - if err := admin.client.DeleteTable(ctx, table.Name); err != nil { - log.Printf("Could not delete table %s err %s", table, err) - return err - } else { - log.Printf("Deleted Table: %v", table.Name) - } - } - return nil -} From d76c0fb1467e6a3475e13614f744fde079f3ea4f Mon Sep 17 00:00:00 2001 From: peter <1674920+peterbitfly@users.noreply.github.com> Date: Fri, 1 Sep 2023 10:55:59 +0200 Subject: [PATCH 068/106] (BIDS-2379) fix build error --- cmd/notification-sender/main.go | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/cmd/notification-sender/main.go b/cmd/notification-sender/main.go index 6a9e950dfd..e253a94f2e 100644 --- a/cmd/notification-sender/main.go +++ b/cmd/notification-sender/main.go @@ -105,6 +105,10 @@ func main() { db.BigtableClient = bt }() + if utils.Config.TieredCacheProvider != "redis" { + logrus.Fatalf("no cache provider set, please set TierdCacheProvider (redis)") + } + if utils.Config.TieredCacheProvider == "redis" || len(utils.Config.RedisCacheEndpoint) != 0 { wg.Add(1) go func() { @@ -115,14 +119,6 @@ func main() { } wg.Wait() - if utils.Config.TieredCacheProvider == "bigtable" && len(utils.Config.RedisCacheEndpoint) == 0 { - cache.MustInitTieredCacheBigtable(db.BigtableClient.GetClient(), fmt.Sprintf("%d", utils.Config.Chain.Config.DepositChainID)) - logrus.Infof("tiered Cache initialized, latest finalized epoch: %v", services.LatestFinalizedEpoch()) - } - - if utils.Config.TieredCacheProvider != "bigtable" && utils.Config.TieredCacheProvider != "redis" { - logrus.Fatalf("no cache provider set, please set TierdCacheProvider (example redis, bigtable)") - } defer db.ReaderDb.Close() defer db.WriterDb.Close() From 73a26f2bb657340556225eb857a75f250ffeb2fc Mon Sep 17 00:00:00 2001 From: peter <1674920+peterbitfly@users.noreply.github.com> Date: Fri, 1 Sep 2023 10:56:26 +0200 Subject: [PATCH 069/106] (BIDS-2379) fix build error --- cmd/notification-collector/main.go | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/cmd/notification-collector/main.go b/cmd/notification-collector/main.go index 3d84e29f3d..653c659598 100644 --- a/cmd/notification-collector/main.go +++ b/cmd/notification-collector/main.go @@ -105,6 +105,9 @@ func main() { db.BigtableClient = bt }() + if utils.Config.TieredCacheProvider != "redis" { + logrus.Fatalf("no cache provider set, please set TierdCacheProvider (redis)") + } if utils.Config.TieredCacheProvider == "redis" || len(utils.Config.RedisCacheEndpoint) != 0 { wg.Add(1) go func() { @@ -115,14 +118,6 @@ func main() { } wg.Wait() - if utils.Config.TieredCacheProvider == "bigtable" && len(utils.Config.RedisCacheEndpoint) == 0 { - cache.MustInitTieredCacheBigtable(db.BigtableClient.GetClient(), fmt.Sprintf("%d", utils.Config.Chain.Config.DepositChainID)) - logrus.Infof("tiered Cache initialized, latest finalized epoch: %v", services.LatestFinalizedEpoch()) - } - - if utils.Config.TieredCacheProvider != "bigtable" && utils.Config.TieredCacheProvider != "redis" { - logrus.Fatalf("no cache provider set, please set TierdCacheProvider (example redis, bigtable)") - } defer db.ReaderDb.Close() defer db.WriterDb.Close() From cead20fde375e82f00e46f239dd3967d1aaecb9c Mon Sep 17 00:00:00 2001 From: peter <1674920+peterbitfly@users.noreply.github.com> Date: Mon, 4 Sep 2023 11:16:45 +0200 Subject: [PATCH 070/106] (BIDS-2379) simplify bigtable schema init in local deployments --- cmd/misc/main.go | 9 +- db/bigtable.go | 8 +- db/bigtable_init.go | 98 +++++++++++++++++++ local-deployment/README.md | 10 -- local-deployment/explorer-config-template.yml | 62 ++++++++++++ local-deployment/main.star | 48 +++++++++ local-deployment/provision-explorer-config.sh | 53 +--------- types/config.go | 1 + 8 files changed, 224 insertions(+), 65 deletions(-) create mode 100644 db/bigtable_init.go create mode 100644 local-deployment/explorer-config-template.yml diff --git a/cmd/misc/main.go b/cmd/misc/main.go index 7669cbf2b6..ef528988f0 100644 --- a/cmd/misc/main.go +++ b/cmd/misc/main.go @@ -47,7 +47,7 @@ var opts = struct { func main() { configPath := flag.String("config", "config/default.config.yml", "Path to the config file") - flag.StringVar(&opts.Command, "command", "", "command to run, available: updateAPIKey, applyDbSchema, epoch-export, debug-rewards, clear-bigtable, index-old-eth1-blocks, update-aggregation-bits, historic-prices-export, index-missing-blocks, export-epoch-missed-slots, migrate-last-attestation-slot-bigtable, generate-config-from-testnet-stub") + flag.StringVar(&opts.Command, "command", "", "command to run, available: updateAPIKey, applyDbSchema, initBigtableSchema, epoch-export, debug-rewards, clear-bigtable, index-old-eth1-blocks, update-aggregation-bits, historic-prices-export, index-missing-blocks, export-epoch-missed-slots, migrate-last-attestation-slot-bigtable, generate-config-from-testnet-stub") flag.Uint64Var(&opts.StartEpoch, "start-epoch", 0, "start epoch") flag.Uint64Var(&opts.EndEpoch, "end-epoch", 0, "end epoch") flag.Uint64Var(&opts.User, "user", 0, "user id") @@ -151,6 +151,13 @@ func main() { logrus.WithError(err).Fatal("error applying db schema") } logrus.Infof("db schema applied successfully") + case "initBigtableSchema": + logrus.Infof("initializing bigtable schema") + err := db.InitBigtableSchema() + if err != nil { + logrus.WithError(err).Fatal("error initializing bigtable schema") + } + logrus.Infof("bigtable schema initialization completed") case "epoch-export": logrus.Infof("exporting epochs %v - %v", opts.StartEpoch, opts.EndEpoch) diff --git a/db/bigtable.go b/db/bigtable.go index a5e3fe5ef9..e1613bfe20 100644 --- a/db/bigtable.go +++ b/db/bigtable.go @@ -74,8 +74,12 @@ type Bigtable struct { func InitBigtable(project, instance, chainId, redisAddress string) (*Bigtable, error) { if utils.Config.Bigtable.Emulator { - logger.Infof("using emulated local bigtable environment, setting BIGTABLE_EMULATOR_HOST env variable to 127.0.0.1:%d", utils.Config.Bigtable.EmulatorPort) - err := os.Setenv("BIGTABLE_EMULATOR_HOST", fmt.Sprintf("127.0.0.1:%d", utils.Config.Bigtable.EmulatorPort)) + + if utils.Config.Bigtable.EmulatorHost == "" { + utils.Config.Bigtable.EmulatorHost = "127.0.0.1" + } + logger.Infof("using emulated local bigtable environment, setting BIGTABLE_EMULATOR_HOST env variable to %s:%d", utils.Config.Bigtable.EmulatorHost, utils.Config.Bigtable.EmulatorPort) + err := os.Setenv("BIGTABLE_EMULATOR_HOST", fmt.Sprintf("%s:%d", utils.Config.Bigtable.EmulatorHost, utils.Config.Bigtable.EmulatorPort)) if err != nil { logger.Fatalf("unable to set bigtable emulator environment variable: %v", err) diff --git a/db/bigtable_init.go b/db/bigtable_init.go new file mode 100644 index 0000000000..56536236d5 --- /dev/null +++ b/db/bigtable_init.go @@ -0,0 +1,98 @@ +package db + +import ( + "context" + "eth2-exporter/utils" + "fmt" + "time" + + gcp_bigtable "cloud.google.com/go/bigtable" +) + +func InitBigtableSchema() error { + + tables := make(map[string]map[string]gcp_bigtable.GCPolicy) + + tables["beaconchain_validator_balances"] = map[string]gcp_bigtable.GCPolicy{ + VALIDATOR_BALANCES_FAMILY: nil, + VALIDATOR_HIGHEST_ACTIVE_INDEX_FAMILY: nil, + } + tables["beaconchain_validator_attestations"] = map[string]gcp_bigtable.GCPolicy{ + ATTESTATIONS_FAMILY: nil, + } + tables["beaconchain_validator_proposals"] = map[string]gcp_bigtable.GCPolicy{ + PROPOSALS_FAMILY: nil, + } + tables["beaconchain_validator_sync"] = map[string]gcp_bigtable.GCPolicy{ + SYNC_COMMITTEES_FAMILY: nil, + SYNC_COMMITTEES_PARTICIPATION_FAMILY: nil, + } + tables["beaconchain_validator_income"] = map[string]gcp_bigtable.GCPolicy{ + INCOME_DETAILS_COLUMN_FAMILY: nil, + STATS_COLUMN_FAMILY: nil, + } + tables["beaconchain_validators"] = map[string]gcp_bigtable.GCPolicy{ + ATTESTATIONS_FAMILY: gcp_bigtable.MaxVersionsGCPolicy(1), + } + tables["blocks"] = map[string]gcp_bigtable.GCPolicy{ + DEFAULT_FAMILY_BLOCKS: gcp_bigtable.MaxVersionsGCPolicy(1), + } + tables["data"] = map[string]gcp_bigtable.GCPolicy{ + CONTRACT_METADATA_FAMILY: gcp_bigtable.MaxAgeGCPolicy(time.Hour * 24), + DEFAULT_FAMILY: nil, + } + tables["machine_metrics"] = map[string]gcp_bigtable.GCPolicy{ + MACHINE_METRICS_COLUMN_FAMILY: gcp_bigtable.MaxAgeGCPolicy(time.Hour * 24 * 31), + } + tables["metadata"] = map[string]gcp_bigtable.GCPolicy{ + ACCOUNT_METADATA_FAMILY: nil, + CONTRACT_METADATA_FAMILY: nil, + ERC20_METADATA_FAMILY: nil, + ERC721_METADATA_FAMILY: nil, + ERC1155_METADATA_FAMILY: nil, + SERIES_FAMILY: gcp_bigtable.MaxVersionsGCPolicy(1), + } + tables["metadata_updates"] = map[string]gcp_bigtable.GCPolicy{ + METADATA_UPDATES_FAMILY_BLOCKS: gcp_bigtable.MaxAgeGCPolicy(time.Hour * 24), + DEFAULT_FAMILY: nil, + } + ctx, cancel := context.WithTimeout(context.Background(), time.Second*30) + defer cancel() + + admin, err := gcp_bigtable.NewAdminClient(ctx, utils.Config.Bigtable.Project, utils.Config.Bigtable.Instance) + if err != nil { + return err + } + + existingTables, err := admin.Tables(ctx) + if err != nil { + return err + } + + if len(existingTables) > 0 { + return fmt.Errorf("aborting bigtable schema init as tables are already present") + } + + for name, definition := range tables { + err := admin.CreateTable(ctx, name) + if err != nil { + return err + } + + for columnFamily, gcPolicy := range definition { + err := admin.CreateColumnFamily(ctx, name, columnFamily) + if err != nil { + return err + } + + if gcPolicy != nil { + err := admin.SetGCPolicy(ctx, name, columnFamily, gcPolicy) + if err != nil { + return err + } + } + } + } + + return nil +} diff --git a/local-deployment/README.md b/local-deployment/README.md index ec467dd5c8..68f5feb54b 100644 --- a/local-deployment/README.md +++ b/local-deployment/README.md @@ -31,16 +31,6 @@ Add the golang binaries to the path by adding the following lines to your ~/.pro export PATH=$PATH:/usr/local/go/bin export PATH=$PATH:$HOME/go/bin ``` -Create the testnet directory -``` -mkdir testnet -cd testnet -``` -# Install the cbt tool -``` -sudo apt remove google-cloud-cli -sudo apt install google-cloud-sdk-cbt -``` # Clone the explorer repository ``` cd ~/ diff --git a/local-deployment/explorer-config-template.yml b/local-deployment/explorer-config-template.yml new file mode 100644 index 0000000000..6b35d27f16 --- /dev/null +++ b/local-deployment/explorer-config-template.yml @@ -0,0 +1,62 @@ +chain: + configPath: 'node' +readerDatabase: + name: db + host: {{.DBHost}} + port: {{.DBPort}} + user: postgres + password: "pass" +writerDatabase: + name: db + host: {{.DBHost}} + port: {{.DBPort}} + user: postgres + password: "pass" +bigtable: + project: explorer + instance: explorer + emulator: true + emulatorHost: {{.LBTHost}} + emulatorPort: {{.LBTPort}} +eth1ErigonEndpoint: '{{.ELNodeEndpoint}}' +eth1GethEndpoint: '{{.ELNodeEndpoint}}' +redisCacheEndpoint: '{{.RedisEndpoint}}' +tieredCacheProvider: 'redis' +frontend: + siteDomain: "localhost:8080" + siteName: 'Open Source Ethereum (ETH) Testnet Explorer' # Name of the site, displayed in the title tag + siteSubtitle: "Showing a local testnet." + server: + host: '0.0.0.0' # Address to listen on + port: '8080' # Port to listen on + readerDatabase: + name: db + host: {{.DBHost}} + port: {{.DBPort}} + user: postgres + password: "pass" + writerDatabase: + name: db + host: {{.DBHost}} + port: {{.DBPort}} + user: postgres + password: "pass" + sessionSecret: "11111111111111111111111111111111" + jwtSigningSecret: "1111111111111111111111111111111111111111111111111111111111111111" + jwtIssuer: "localhost" + jwtValidityInMinutes: 30 + maxMailsPerEmailPerDay: 10 + mail: + mailgun: + sender: no-reply@localhost + domain: mg.localhost + privateKey: "key-11111111111111111111111111111111" + csrfAuthKey: '1111111111111111111111111111111111111111111111111111111111111111' +indexer: + # fullIndexOnStartup: false # Perform a one time full db index on startup + # indexMissingEpochsOnStartup: true # Check for missing epochs and export them after startup + node: + host: '{{.CLNodeHost}}' + port: '{{.CLNodePort}}' + type: lighthouse + eth1DepositContractFirstBlock: 0 diff --git a/local-deployment/main.star b/local-deployment/main.star index e33e7cf18a..b826a77f39 100644 --- a/local-deployment/main.star +++ b/local-deployment/main.star @@ -2,6 +2,7 @@ parse_input = import_module("github.com/kurtosis-tech/eth2-package/src/package_i eth_network_module = import_module("github.com/kurtosis-tech/eth-network-package/main.star") transaction_spammer = import_module("github.com/kurtosis-tech/eth2-package/src/transaction_spammer/transaction_spammer.star") genesis_constants = import_module("github.com/kurtosis-tech/eth-network-package/src/prelaunch_data_generator/genesis_constants/genesis_constants.star") +shared_utils = import_module("github.com/kurtosis-tech/eth2-package/src/shared_utils/shared_utils.star") POSTGRES_PORT_ID = "postgres" POSTGRES_DB = "db" @@ -12,6 +13,8 @@ REDIS_PORT_ID = "redis" LITTLE_BIGTABLE_PORT_ID = "littlebigtable" +EXPLORER_CONFIG_FILENAME = "config.yml" + def run(plan, args): # Add a Postgres server postgres = plan.add_service( @@ -61,3 +64,48 @@ def run(plan, args): # transaction_spammer.launch_transaction_spammer(plan, genesis_constants.PRE_FUNDED_ACCOUNTS, all_el_client_contexts[0]) # plan.print("Succesfully launched transaction spammer") + # Still WIP: + # el_uri = "http://{0}:{1}".format(all_el_client_contexts[0].ip_addr, all_el_client_contexts[0].rpc_port_num) + # redis_uri = "{0}:{1}".format(redis.ip_address, 6379) + + # plan.print("{0}".format(all_cl_client_contexts[0].ip_addr)) + + # config_template = read_file("./explorer-config-template.yml") + # template_data = new_config_template_data(all_cl_client_contexts[0], el_uri, littlebigtable.ip_address, 9000, postgres.ip_address, 5432, redis_uri) + # template_and_data = shared_utils.new_template_and_data(config_template, template_data) + # template_and_data_by_rel_dest_filepath = {} + # template_and_data_by_rel_dest_filepath[EXPLORER_CONFIG_FILENAME] = template_and_data + + # config_files_artifact_name = plan.render_templates(template_and_data_by_rel_dest_filepath, "config.yml") + + # createdbschema = plan.add_service( + # name = "createdbschema", + # config = ServiceConfig( + # image = "explorer", + # files = { + # "/app/config/": config_files_artifact_name, + # }, + # entrypoint = [ + # "./misc" + # ], + # cmd = [ + # "-config", + # "/app/config/config.yml", + # "-command", + # "applyDbSchema" + # ], + # ), + # ) + +def new_config_template_data(cl_node_info, el_uri, lbt_host, lbt_port, db_host, db_port, redis_uri): + return { + "CLNodeHost": cl_node_info.ip_addr, + "CLNodePort": cl_node_info.http_port_num, + "ELNodeEndpoint": el_uri, + "LBTHost": lbt_host, + "LBTPort": lbt_port, + "DBHost": db_host, + "DBPort": db_port, + "RedisEndpoint": redis_uri, + + } \ No newline at end of file diff --git a/local-deployment/provision-explorer-config.sh b/local-deployment/provision-explorer-config.sh index da2f9b59b0..3aeed31dbb 100644 --- a/local-deployment/provision-explorer-config.sh +++ b/local-deployment/provision-explorer-config.sh @@ -85,58 +85,7 @@ PROJECT="explorer" INSTANCE="explorer" HOST="127.0.0.1:$LBT_PORT" -BIGTABLE_EMULATOR_HOST=$HOST cbt --project $PROJECT --instance $INSTANCE createtable beaconchain_validator_balances -BIGTABLE_EMULATOR_HOST=$HOST cbt --project $PROJECT --instance $INSTANCE createfamily beaconchain_validator_balances vb -BIGTABLE_EMULATOR_HOST=$HOST cbt --project $PROJECT --instance $INSTANCE createfamily beaconchain_validator_balances ha -BIGTABLE_EMULATOR_HOST=$HOST cbt --project $PROJECT --instance $INSTANCE createtable beaconchain_validator_attestations -BIGTABLE_EMULATOR_HOST=$HOST cbt --project $PROJECT --instance $INSTANCE createfamily beaconchain_validator_attestations at -BIGTABLE_EMULATOR_HOST=$HOST cbt --project $PROJECT --instance $INSTANCE createtable beaconchain_validator_proposals -BIGTABLE_EMULATOR_HOST=$HOST cbt --project $PROJECT --instance $INSTANCE createfamily beaconchain_validator_proposals pr -BIGTABLE_EMULATOR_HOST=$HOST cbt --project $PROJECT --instance $INSTANCE createtable beaconchain_validator_sync -BIGTABLE_EMULATOR_HOST=$HOST cbt --project $PROJECT --instance $INSTANCE createfamily beaconchain_validator_sync sc -BIGTABLE_EMULATOR_HOST=$HOST cbt --project $PROJECT --instance $INSTANCE createfamily beaconchain_validator_sync sp -BIGTABLE_EMULATOR_HOST=$HOST cbt --project $PROJECT --instance $INSTANCE createtable beaconchain_validator_income -BIGTABLE_EMULATOR_HOST=$HOST cbt --project $PROJECT --instance $INSTANCE createfamily beaconchain_validator_income id -BIGTABLE_EMULATOR_HOST=$HOST cbt --project $PROJECT --instance $INSTANCE createfamily beaconchain_validator_income stats -BIGTABLE_EMULATOR_HOST=$HOST cbt --project $PROJECT --instance $INSTANCE createtable beaconchain_validators -BIGTABLE_EMULATOR_HOST=$HOST cbt --project $PROJECT --instance $INSTANCE createfamily beaconchain_validators at - -BIGTABLE_EMULATOR_HOST=$HOST cbt --project $PROJECT --instance $INSTANCE setgcpolicy beaconchain_validators at maxversions=1 -BIGTABLE_EMULATOR_HOST=$HOST cbt --project $PROJECT --instance $INSTANCE createtable blocks -BIGTABLE_EMULATOR_HOST=$HOST cbt --project $PROJECT --instance $INSTANCE createfamily blocks default - -BIGTABLE_EMULATOR_HOST=$HOST cbt --project $PROJECT --instance $INSTANCE setgcpolicy blocks default maxversions=1 -BIGTABLE_EMULATOR_HOST=$HOST cbt --project $PROJECT --instance $INSTANCE createtable cache -BIGTABLE_EMULATOR_HOST=$HOST cbt --project $PROJECT --instance $INSTANCE createfamily cache 10_min -BIGTABLE_EMULATOR_HOST=$HOST cbt --project $PROJECT --instance $INSTANCE createfamily cache 1_day -BIGTABLE_EMULATOR_HOST=$HOST cbt --project $PROJECT --instance $INSTANCE createfamily cache 1_hour - -BIGTABLE_EMULATOR_HOST=$HOST cbt --project $PROJECT --instance $INSTANCE setgcpolicy cache 10_min maxage=10m and maxversions=1 -BIGTABLE_EMULATOR_HOST=$HOST cbt --project $PROJECT --instance $INSTANCE setgcpolicy cache 1_day maxage=1d and maxversions=1 -BIGTABLE_EMULATOR_HOST=$HOST cbt --project $PROJECT --instance $INSTANCE setgcpolicy cache 1_hour maxage=1h and maxversions=1 -BIGTABLE_EMULATOR_HOST=$HOST cbt --project $PROJECT --instance $INSTANCE createtable data -BIGTABLE_EMULATOR_HOST=$HOST cbt --project $PROJECT --instance $INSTANCE createfamily data c -BIGTABLE_EMULATOR_HOST=$HOST cbt --project $PROJECT --instance $INSTANCE createfamily data f - -BIGTABLE_EMULATOR_HOST=$HOST cbt --project $PROJECT --instance $INSTANCE setgcpolicy data c maxage=1d -BIGTABLE_EMULATOR_HOST=$HOST cbt --project $PROJECT --instance $INSTANCE createtable machine_metrics -BIGTABLE_EMULATOR_HOST=$HOST cbt --project $PROJECT --instance $INSTANCE createfamily machine_metrics mm - -BIGTABLE_EMULATOR_HOST=$HOST cbt --project $PROJECT --instance $INSTANCE setgcpolicy machine_metrics mm maxage=31d -BIGTABLE_EMULATOR_HOST=$HOST cbt --project $PROJECT --instance $INSTANCE createtable metadata -BIGTABLE_EMULATOR_HOST=$HOST cbt --project $PROJECT --instance $INSTANCE createfamily metadata a -BIGTABLE_EMULATOR_HOST=$HOST cbt --project $PROJECT --instance $INSTANCE createfamily metadata c -BIGTABLE_EMULATOR_HOST=$HOST cbt --project $PROJECT --instance $INSTANCE createfamily metadata erc1155 -BIGTABLE_EMULATOR_HOST=$HOST cbt --project $PROJECT --instance $INSTANCE createfamily metadata erc20 -BIGTABLE_EMULATOR_HOST=$HOST cbt --project $PROJECT --instance $INSTANCE createfamily metadata erc721 -BIGTABLE_EMULATOR_HOST=$HOST cbt --project $PROJECT --instance $INSTANCE createfamily metadata series - -BIGTABLE_EMULATOR_HOST=$HOST cbt --project $PROJECT --instance $INSTANCE setgcpolicy metadata series maxversions=1 -BIGTABLE_EMULATOR_HOST=$HOST cbt --project $PROJECT --instance $INSTANCE createtable metadata_updates -BIGTABLE_EMULATOR_HOST=$HOST cbt --project $PROJECT --instance $INSTANCE createfamily metadata_updates blocks -BIGTABLE_EMULATOR_HOST=$HOST cbt --project $PROJECT --instance $INSTANCE createfamily metadata_updates f - -BIGTABLE_EMULATOR_HOST=$HOST cbt --project $PROJECT --instance $INSTANCE setgcpolicy metadata_updates blocks maxage=1d +go run ../cmd/misc/main.go -config config.yml -command initBigtableSchema echo "bigtable schema initialization completed" diff --git a/types/config.go b/types/config.go index f3b74674a7..75455b5919 100644 --- a/types/config.go +++ b/types/config.go @@ -30,6 +30,7 @@ type Config struct { Instance string `yaml:"instance" envconfig:"BIGTABLE_INSTANCE"` Emulator bool `yaml:"emulator" envconfig:"BIGTABLE_EMULATOR"` EmulatorPort int `yaml:"emulatorPort" envconfig:"BIGTABLE_EMULATOR_PORT"` + EmulatorHost string `yaml:"emulatorHost" envconfig:"BIGTABLE_EMULATOR_HOST"` } `yaml:"bigtable"` Chain struct { Name string `yaml:"name" envconfig:"CHAIN_NAME"` From b5869f960a470c65e6290eab28fba3fa17899461 Mon Sep 17 00:00:00 2001 From: peter <1674920+peterbitfly@users.noreply.github.com> Date: Mon, 4 Sep 2023 11:52:58 +0200 Subject: [PATCH 071/106] (BIDS-2379) add standalone kurtosis module --- .../kurtosis/explorer-config-template.yml | 62 +++++ local-deployment/kurtosis/kurtosis.yml | 1 + local-deployment/kurtosis/main.star | 254 ++++++++++++++++++ local-deployment/kurtosis/network-params.json | 29 ++ local-deployment/main.star | 1 + 5 files changed, 347 insertions(+) create mode 100644 local-deployment/kurtosis/explorer-config-template.yml create mode 100644 local-deployment/kurtosis/kurtosis.yml create mode 100644 local-deployment/kurtosis/main.star create mode 100644 local-deployment/kurtosis/network-params.json diff --git a/local-deployment/kurtosis/explorer-config-template.yml b/local-deployment/kurtosis/explorer-config-template.yml new file mode 100644 index 0000000000..6b35d27f16 --- /dev/null +++ b/local-deployment/kurtosis/explorer-config-template.yml @@ -0,0 +1,62 @@ +chain: + configPath: 'node' +readerDatabase: + name: db + host: {{.DBHost}} + port: {{.DBPort}} + user: postgres + password: "pass" +writerDatabase: + name: db + host: {{.DBHost}} + port: {{.DBPort}} + user: postgres + password: "pass" +bigtable: + project: explorer + instance: explorer + emulator: true + emulatorHost: {{.LBTHost}} + emulatorPort: {{.LBTPort}} +eth1ErigonEndpoint: '{{.ELNodeEndpoint}}' +eth1GethEndpoint: '{{.ELNodeEndpoint}}' +redisCacheEndpoint: '{{.RedisEndpoint}}' +tieredCacheProvider: 'redis' +frontend: + siteDomain: "localhost:8080" + siteName: 'Open Source Ethereum (ETH) Testnet Explorer' # Name of the site, displayed in the title tag + siteSubtitle: "Showing a local testnet." + server: + host: '0.0.0.0' # Address to listen on + port: '8080' # Port to listen on + readerDatabase: + name: db + host: {{.DBHost}} + port: {{.DBPort}} + user: postgres + password: "pass" + writerDatabase: + name: db + host: {{.DBHost}} + port: {{.DBPort}} + user: postgres + password: "pass" + sessionSecret: "11111111111111111111111111111111" + jwtSigningSecret: "1111111111111111111111111111111111111111111111111111111111111111" + jwtIssuer: "localhost" + jwtValidityInMinutes: 30 + maxMailsPerEmailPerDay: 10 + mail: + mailgun: + sender: no-reply@localhost + domain: mg.localhost + privateKey: "key-11111111111111111111111111111111" + csrfAuthKey: '1111111111111111111111111111111111111111111111111111111111111111' +indexer: + # fullIndexOnStartup: false # Perform a one time full db index on startup + # indexMissingEpochsOnStartup: true # Check for missing epochs and export them after startup + node: + host: '{{.CLNodeHost}}' + port: '{{.CLNodePort}}' + type: lighthouse + eth1DepositContractFirstBlock: 0 diff --git a/local-deployment/kurtosis/kurtosis.yml b/local-deployment/kurtosis/kurtosis.yml new file mode 100644 index 0000000000..31a6a673bb --- /dev/null +++ b/local-deployment/kurtosis/kurtosis.yml @@ -0,0 +1 @@ +name: "github.com/gobitfly/eth2-beaconchain-explorer/local-testnet/kurtosis" \ No newline at end of file diff --git a/local-deployment/kurtosis/main.star b/local-deployment/kurtosis/main.star new file mode 100644 index 0000000000..fb7e4abe01 --- /dev/null +++ b/local-deployment/kurtosis/main.star @@ -0,0 +1,254 @@ +parse_input = import_module("github.com/kurtosis-tech/eth2-package/src/package_io/parse_input.star") +eth_network_module = import_module("github.com/kurtosis-tech/eth-network-package/main.star") +transaction_spammer = import_module("github.com/kurtosis-tech/eth2-package/src/transaction_spammer/transaction_spammer.star") +genesis_constants = import_module("github.com/kurtosis-tech/eth-network-package/src/prelaunch_data_generator/genesis_constants/genesis_constants.star") +shared_utils = import_module("github.com/kurtosis-tech/eth2-package/src/shared_utils/shared_utils.star") + +POSTGRES_PORT_ID = "postgres" +POSTGRES_DB = "db" +POSTGRES_USER = "postgres" +POSTGRES_PASSWORD = "pass" + +REDIS_PORT_ID = "redis" + +FRONTEND_PORT_ID = "frontend" + +LITTLE_BIGTABLE_PORT_ID = "littlebigtable" + +EXPLORER_CONFIG_FILENAME = "config.yml" + +def run(plan, args): + # Add a Postgres server + postgres = plan.add_service( + name = "postgres", + config = ServiceConfig( + image = "postgres:15.2-alpine", + ports = { + POSTGRES_PORT_ID: PortSpec(5432, application_protocol = "postgresql"), + }, + env_vars = { + "POSTGRES_DB": POSTGRES_DB, + "POSTGRES_USER": POSTGRES_USER, + "POSTGRES_PASSWORD": POSTGRES_PASSWORD, + }, + ), + ) + # Add a redis server + redis = plan.add_service( + name = "redis", + config = ServiceConfig( + image = "redis:7", + ports = { + REDIS_PORT_ID: PortSpec(6379, application_protocol = "tcp"), + }, + ), + ) + # Add a little bigtable server + littlebigtable = plan.add_service( + name = "littlebigtable", + config = ServiceConfig( + image = "gobitfly/little_bigtable:latest", + ports = { + LITTLE_BIGTABLE_PORT_ID: PortSpec(9000, application_protocol = "tcp"), + }, + ), + ) + + # Spin up a local ethereum testnet + all_participants, cl_genesis_timestamp, genesis_validators_root = eth_network_module.run(plan, args) + + all_el_client_contexts = [] + all_cl_client_contexts = [] + for participant in all_participants: + all_el_client_contexts.append(participant.el_client_context) + all_cl_client_contexts.append(participant.cl_client_context) + + # plan.print("Launching transaction spammer") + # transaction_spammer.launch_transaction_spammer(plan, genesis_constants.PRE_FUNDED_ACCOUNTS, all_el_client_contexts[0]) + # plan.print("Succesfully launched transaction spammer") + + el_uri = "http://{0}:{1}".format(all_el_client_contexts[0].ip_addr, all_el_client_contexts[0].rpc_port_num) + redis_uri = "{0}:{1}".format(redis.ip_address, 6379) + + plan.print("{0}".format(all_cl_client_contexts[0].ip_addr)) + + config_template = read_file("./explorer-config-template.yml") + template_data = new_config_template_data(all_cl_client_contexts[0], el_uri, littlebigtable.ip_address, 9000, postgres.ip_address, 5432, redis_uri) + template_and_data = shared_utils.new_template_and_data(config_template, template_data) + template_and_data_by_rel_dest_filepath = {} + template_and_data_by_rel_dest_filepath[EXPLORER_CONFIG_FILENAME] = template_and_data + + config_files_artifact_name = plan.render_templates(template_and_data_by_rel_dest_filepath, "config.yml") + + # Initialize the db schema + initdbschema = plan.add_service( + name = "initdbschema", + config = ServiceConfig( + image = "gobitfly/eth2-beaconchain-explorer:kurtosis", + files = { + "/app/config/": config_files_artifact_name, + }, + entrypoint = [ + "./misc" + ], + cmd = [ + "-config", + "/app/config/config.yml", + "-command", + "applyDbSchema" + ], + ), + ) + # Initialize the bigtable schema + initbigtableschema = plan.add_service( + name = "initbigtableschema", + config = ServiceConfig( + image = "gobitfly/eth2-beaconchain-explorer:kurtosis", + files = { + "/app/config/": config_files_artifact_name, + }, + entrypoint = [ + "./misc" + ], + cmd = [ + "-config", + "/app/config/config.yml", + "-command", + "initBigtableSchema" + ], + ), + ) + # Start the indexer + indexer = plan.add_service( + name = "indexer", + config = ServiceConfig( + image = "gobitfly/eth2-beaconchain-explorer:kurtosis", + files = { + "/app/config/": config_files_artifact_name, + }, + entrypoint = [ + "./explorer" + ], + cmd = [ + "-config", + "/app/config/config.yml", + ], + env_vars = { + "INDEXER_ENABLED": "TRUE", + } + ), + ) + # Start the eth1indexer + eth1indexer = plan.add_service( + name = "eth1indexer", + config = ServiceConfig( + image = "gobitfly/eth2-beaconchain-explorer:kurtosis", + files = { + "/app/config/": config_files_artifact_name, + }, + entrypoint = [ + "./eth1indexer" + ], + cmd = [ + "-config", + "/app/config/config.yml", + "-blocks.concurrency", + "1", + "-blocks.tracemode", + "geth", + "-data.concurrency", + "1", + "-balances.enabled" + ], + ), + ) + + rewardsexporter = plan.add_service( + name = "rewardsexporter", + config = ServiceConfig( + image = "gobitfly/eth2-beaconchain-explorer:kurtosis", + files = { + "/app/config/": config_files_artifact_name, + }, + entrypoint = [ + "./rewards-exporter" + ], + cmd = [ + "-config", + "/app/config/config.yml", + ], + ), + ) + + statistics = plan.add_service( + name = "statistics", + config = ServiceConfig( + image = "gobitfly/eth2-beaconchain-explorer:kurtosis", + files = { + "/app/config/": config_files_artifact_name, + }, + entrypoint = [ + "./statistics" + ], + cmd = [ + "-config", + "/app/config/config.yml", + "-charts.enabled", + "-graffiti.enabled", + "-validators.enabled" + ], + ), + ) + + fdu = plan.add_service( + name = "fdu", + config = ServiceConfig( + image = "gobitfly/eth2-beaconchain-explorer:kurtosis", + files = { + "/app/config/": config_files_artifact_name, + }, + entrypoint = [ + "./frontend-data-updater" + ], + cmd = [ + "-config", + "/app/config/config.yml", + ], + ), + ) + + frontend = plan.add_service( + name = "frontend", + config = ServiceConfig( + image = "gobitfly/eth2-beaconchain-explorer:kurtosis", + files = { + "/app/config/": config_files_artifact_name, + }, + entrypoint = [ + "./explorer" + ], + cmd = [ + "-config", + "/app/config/config.yml", + ], + env_vars = { + "FRONTEND_ENABLED": "TRUE", + }, + ports = { + FRONTEND_PORT_ID: PortSpec(8080, application_protocol = "http"), + }, + ), + ) + +def new_config_template_data(cl_node_info, el_uri, lbt_host, lbt_port, db_host, db_port, redis_uri): + return { + "CLNodeHost": cl_node_info.ip_addr, + "CLNodePort": cl_node_info.http_port_num, + "ELNodeEndpoint": el_uri, + "LBTHost": lbt_host, + "LBTPort": lbt_port, + "DBHost": db_host, + "DBPort": db_port, + "RedisEndpoint": redis_uri, + + } \ No newline at end of file diff --git a/local-deployment/kurtosis/network-params.json b/local-deployment/kurtosis/network-params.json new file mode 100644 index 0000000000..2c7231f300 --- /dev/null +++ b/local-deployment/kurtosis/network-params.json @@ -0,0 +1,29 @@ +{ + "participants": [ + { + "el_client_type": "geth", + "el_client_image": "", + "el_client_log_level": "", + "cl_client_type": "lighthouse", + "cl_client_image": "", + "cl_client_log_level": "", + "beacon_extra_params": [], + "el_extra_params": ["--http.api='admin,eth,net,debug,txpool' --syncmode full --gcmode archive"], + "validator_extra_params": [], + "builder_network_params": null, + "count": 1 + } + ], + "network_params": { + "preregistered_validator_keys_mnemonic": "giant issue aisle success illegal bike spike question tent bar rely arctic volcano long crawl hungry vocal artwork sniff fantasy very lucky have athlete", + "num_validator_keys_per_node": 64, + "network_id": "3151908", + "deposit_contract_address": "0x4242424242424242424242424242424242424242", + "seconds_per_slot": 4, + "genesis_delay": 10, + "capella_fork_epoch": 5, + "deneb_fork_epoch": 500 + }, + "global_client_log_level": "info" + } + \ No newline at end of file diff --git a/local-deployment/main.star b/local-deployment/main.star index b826a77f39..7f53db617f 100644 --- a/local-deployment/main.star +++ b/local-deployment/main.star @@ -52,6 +52,7 @@ def run(plan, args): ), ) + # Spin up a local ethereum testnet all_participants, cl_genesis_timestamp, genesis_validators_root = eth_network_module.run(plan, args) all_el_client_contexts = [] From b914bfa025d3ba6ad88c96af5716071af3cccdb9 Mon Sep 17 00:00:00 2001 From: peter <1674920+peterbitfly@users.noreply.github.com> Date: Mon, 4 Sep 2023 13:43:25 +0200 Subject: [PATCH 072/106] (BIDS-2379) fix error for non-existing config.yml file during first local devnet startup --- .../kurtosis/explorer-config-template.yml | 62 ----- local-deployment/kurtosis/kurtosis.yml | 1 - local-deployment/kurtosis/main.star | 254 ------------------ local-deployment/kurtosis/network-params.json | 29 -- local-deployment/provision-explorer-config.sh | 2 + 5 files changed, 2 insertions(+), 346 deletions(-) delete mode 100644 local-deployment/kurtosis/explorer-config-template.yml delete mode 100644 local-deployment/kurtosis/kurtosis.yml delete mode 100644 local-deployment/kurtosis/main.star delete mode 100644 local-deployment/kurtosis/network-params.json diff --git a/local-deployment/kurtosis/explorer-config-template.yml b/local-deployment/kurtosis/explorer-config-template.yml deleted file mode 100644 index 6b35d27f16..0000000000 --- a/local-deployment/kurtosis/explorer-config-template.yml +++ /dev/null @@ -1,62 +0,0 @@ -chain: - configPath: 'node' -readerDatabase: - name: db - host: {{.DBHost}} - port: {{.DBPort}} - user: postgres - password: "pass" -writerDatabase: - name: db - host: {{.DBHost}} - port: {{.DBPort}} - user: postgres - password: "pass" -bigtable: - project: explorer - instance: explorer - emulator: true - emulatorHost: {{.LBTHost}} - emulatorPort: {{.LBTPort}} -eth1ErigonEndpoint: '{{.ELNodeEndpoint}}' -eth1GethEndpoint: '{{.ELNodeEndpoint}}' -redisCacheEndpoint: '{{.RedisEndpoint}}' -tieredCacheProvider: 'redis' -frontend: - siteDomain: "localhost:8080" - siteName: 'Open Source Ethereum (ETH) Testnet Explorer' # Name of the site, displayed in the title tag - siteSubtitle: "Showing a local testnet." - server: - host: '0.0.0.0' # Address to listen on - port: '8080' # Port to listen on - readerDatabase: - name: db - host: {{.DBHost}} - port: {{.DBPort}} - user: postgres - password: "pass" - writerDatabase: - name: db - host: {{.DBHost}} - port: {{.DBPort}} - user: postgres - password: "pass" - sessionSecret: "11111111111111111111111111111111" - jwtSigningSecret: "1111111111111111111111111111111111111111111111111111111111111111" - jwtIssuer: "localhost" - jwtValidityInMinutes: 30 - maxMailsPerEmailPerDay: 10 - mail: - mailgun: - sender: no-reply@localhost - domain: mg.localhost - privateKey: "key-11111111111111111111111111111111" - csrfAuthKey: '1111111111111111111111111111111111111111111111111111111111111111' -indexer: - # fullIndexOnStartup: false # Perform a one time full db index on startup - # indexMissingEpochsOnStartup: true # Check for missing epochs and export them after startup - node: - host: '{{.CLNodeHost}}' - port: '{{.CLNodePort}}' - type: lighthouse - eth1DepositContractFirstBlock: 0 diff --git a/local-deployment/kurtosis/kurtosis.yml b/local-deployment/kurtosis/kurtosis.yml deleted file mode 100644 index 31a6a673bb..0000000000 --- a/local-deployment/kurtosis/kurtosis.yml +++ /dev/null @@ -1 +0,0 @@ -name: "github.com/gobitfly/eth2-beaconchain-explorer/local-testnet/kurtosis" \ No newline at end of file diff --git a/local-deployment/kurtosis/main.star b/local-deployment/kurtosis/main.star deleted file mode 100644 index fb7e4abe01..0000000000 --- a/local-deployment/kurtosis/main.star +++ /dev/null @@ -1,254 +0,0 @@ -parse_input = import_module("github.com/kurtosis-tech/eth2-package/src/package_io/parse_input.star") -eth_network_module = import_module("github.com/kurtosis-tech/eth-network-package/main.star") -transaction_spammer = import_module("github.com/kurtosis-tech/eth2-package/src/transaction_spammer/transaction_spammer.star") -genesis_constants = import_module("github.com/kurtosis-tech/eth-network-package/src/prelaunch_data_generator/genesis_constants/genesis_constants.star") -shared_utils = import_module("github.com/kurtosis-tech/eth2-package/src/shared_utils/shared_utils.star") - -POSTGRES_PORT_ID = "postgres" -POSTGRES_DB = "db" -POSTGRES_USER = "postgres" -POSTGRES_PASSWORD = "pass" - -REDIS_PORT_ID = "redis" - -FRONTEND_PORT_ID = "frontend" - -LITTLE_BIGTABLE_PORT_ID = "littlebigtable" - -EXPLORER_CONFIG_FILENAME = "config.yml" - -def run(plan, args): - # Add a Postgres server - postgres = plan.add_service( - name = "postgres", - config = ServiceConfig( - image = "postgres:15.2-alpine", - ports = { - POSTGRES_PORT_ID: PortSpec(5432, application_protocol = "postgresql"), - }, - env_vars = { - "POSTGRES_DB": POSTGRES_DB, - "POSTGRES_USER": POSTGRES_USER, - "POSTGRES_PASSWORD": POSTGRES_PASSWORD, - }, - ), - ) - # Add a redis server - redis = plan.add_service( - name = "redis", - config = ServiceConfig( - image = "redis:7", - ports = { - REDIS_PORT_ID: PortSpec(6379, application_protocol = "tcp"), - }, - ), - ) - # Add a little bigtable server - littlebigtable = plan.add_service( - name = "littlebigtable", - config = ServiceConfig( - image = "gobitfly/little_bigtable:latest", - ports = { - LITTLE_BIGTABLE_PORT_ID: PortSpec(9000, application_protocol = "tcp"), - }, - ), - ) - - # Spin up a local ethereum testnet - all_participants, cl_genesis_timestamp, genesis_validators_root = eth_network_module.run(plan, args) - - all_el_client_contexts = [] - all_cl_client_contexts = [] - for participant in all_participants: - all_el_client_contexts.append(participant.el_client_context) - all_cl_client_contexts.append(participant.cl_client_context) - - # plan.print("Launching transaction spammer") - # transaction_spammer.launch_transaction_spammer(plan, genesis_constants.PRE_FUNDED_ACCOUNTS, all_el_client_contexts[0]) - # plan.print("Succesfully launched transaction spammer") - - el_uri = "http://{0}:{1}".format(all_el_client_contexts[0].ip_addr, all_el_client_contexts[0].rpc_port_num) - redis_uri = "{0}:{1}".format(redis.ip_address, 6379) - - plan.print("{0}".format(all_cl_client_contexts[0].ip_addr)) - - config_template = read_file("./explorer-config-template.yml") - template_data = new_config_template_data(all_cl_client_contexts[0], el_uri, littlebigtable.ip_address, 9000, postgres.ip_address, 5432, redis_uri) - template_and_data = shared_utils.new_template_and_data(config_template, template_data) - template_and_data_by_rel_dest_filepath = {} - template_and_data_by_rel_dest_filepath[EXPLORER_CONFIG_FILENAME] = template_and_data - - config_files_artifact_name = plan.render_templates(template_and_data_by_rel_dest_filepath, "config.yml") - - # Initialize the db schema - initdbschema = plan.add_service( - name = "initdbschema", - config = ServiceConfig( - image = "gobitfly/eth2-beaconchain-explorer:kurtosis", - files = { - "/app/config/": config_files_artifact_name, - }, - entrypoint = [ - "./misc" - ], - cmd = [ - "-config", - "/app/config/config.yml", - "-command", - "applyDbSchema" - ], - ), - ) - # Initialize the bigtable schema - initbigtableschema = plan.add_service( - name = "initbigtableschema", - config = ServiceConfig( - image = "gobitfly/eth2-beaconchain-explorer:kurtosis", - files = { - "/app/config/": config_files_artifact_name, - }, - entrypoint = [ - "./misc" - ], - cmd = [ - "-config", - "/app/config/config.yml", - "-command", - "initBigtableSchema" - ], - ), - ) - # Start the indexer - indexer = plan.add_service( - name = "indexer", - config = ServiceConfig( - image = "gobitfly/eth2-beaconchain-explorer:kurtosis", - files = { - "/app/config/": config_files_artifact_name, - }, - entrypoint = [ - "./explorer" - ], - cmd = [ - "-config", - "/app/config/config.yml", - ], - env_vars = { - "INDEXER_ENABLED": "TRUE", - } - ), - ) - # Start the eth1indexer - eth1indexer = plan.add_service( - name = "eth1indexer", - config = ServiceConfig( - image = "gobitfly/eth2-beaconchain-explorer:kurtosis", - files = { - "/app/config/": config_files_artifact_name, - }, - entrypoint = [ - "./eth1indexer" - ], - cmd = [ - "-config", - "/app/config/config.yml", - "-blocks.concurrency", - "1", - "-blocks.tracemode", - "geth", - "-data.concurrency", - "1", - "-balances.enabled" - ], - ), - ) - - rewardsexporter = plan.add_service( - name = "rewardsexporter", - config = ServiceConfig( - image = "gobitfly/eth2-beaconchain-explorer:kurtosis", - files = { - "/app/config/": config_files_artifact_name, - }, - entrypoint = [ - "./rewards-exporter" - ], - cmd = [ - "-config", - "/app/config/config.yml", - ], - ), - ) - - statistics = plan.add_service( - name = "statistics", - config = ServiceConfig( - image = "gobitfly/eth2-beaconchain-explorer:kurtosis", - files = { - "/app/config/": config_files_artifact_name, - }, - entrypoint = [ - "./statistics" - ], - cmd = [ - "-config", - "/app/config/config.yml", - "-charts.enabled", - "-graffiti.enabled", - "-validators.enabled" - ], - ), - ) - - fdu = plan.add_service( - name = "fdu", - config = ServiceConfig( - image = "gobitfly/eth2-beaconchain-explorer:kurtosis", - files = { - "/app/config/": config_files_artifact_name, - }, - entrypoint = [ - "./frontend-data-updater" - ], - cmd = [ - "-config", - "/app/config/config.yml", - ], - ), - ) - - frontend = plan.add_service( - name = "frontend", - config = ServiceConfig( - image = "gobitfly/eth2-beaconchain-explorer:kurtosis", - files = { - "/app/config/": config_files_artifact_name, - }, - entrypoint = [ - "./explorer" - ], - cmd = [ - "-config", - "/app/config/config.yml", - ], - env_vars = { - "FRONTEND_ENABLED": "TRUE", - }, - ports = { - FRONTEND_PORT_ID: PortSpec(8080, application_protocol = "http"), - }, - ), - ) - -def new_config_template_data(cl_node_info, el_uri, lbt_host, lbt_port, db_host, db_port, redis_uri): - return { - "CLNodeHost": cl_node_info.ip_addr, - "CLNodePort": cl_node_info.http_port_num, - "ELNodeEndpoint": el_uri, - "LBTHost": lbt_host, - "LBTPort": lbt_port, - "DBHost": db_host, - "DBPort": db_port, - "RedisEndpoint": redis_uri, - - } \ No newline at end of file diff --git a/local-deployment/kurtosis/network-params.json b/local-deployment/kurtosis/network-params.json deleted file mode 100644 index 2c7231f300..0000000000 --- a/local-deployment/kurtosis/network-params.json +++ /dev/null @@ -1,29 +0,0 @@ -{ - "participants": [ - { - "el_client_type": "geth", - "el_client_image": "", - "el_client_log_level": "", - "cl_client_type": "lighthouse", - "cl_client_image": "", - "cl_client_log_level": "", - "beacon_extra_params": [], - "el_extra_params": ["--http.api='admin,eth,net,debug,txpool' --syncmode full --gcmode archive"], - "validator_extra_params": [], - "builder_network_params": null, - "count": 1 - } - ], - "network_params": { - "preregistered_validator_keys_mnemonic": "giant issue aisle success illegal bike spike question tent bar rely arctic volcano long crawl hungry vocal artwork sniff fantasy very lucky have athlete", - "num_validator_keys_per_node": 64, - "network_id": "3151908", - "deposit_contract_address": "0x4242424242424242424242424242424242424242", - "seconds_per_slot": 4, - "genesis_delay": 10, - "capella_fork_epoch": 5, - "deneb_fork_epoch": 500 - }, - "global_client_log_level": "info" - } - \ No newline at end of file diff --git a/local-deployment/provision-explorer-config.sh b/local-deployment/provision-explorer-config.sh index 3aeed31dbb..5ef12a2145 100644 --- a/local-deployment/provision-explorer-config.sh +++ b/local-deployment/provision-explorer-config.sh @@ -14,6 +14,8 @@ echo "Postgres port is $POSTGRES_PORT" LBT_PORT=$(kurtosis enclave inspect my-testnet | grep 9000/tcp | tr -s ' ' | cut -d " " -f 6 | sed -e 's/tcp\:\/\/127.0.0.1\://' | grep "\S") echo "Little bigtable port is $LBT_PORT" +touch config.yml + cat >config.yml < Date: Tue, 5 Sep 2023 09:36:08 +0200 Subject: [PATCH 073/106] (BIDS-2379) speed up startup of local dev instance --- local-deployment/main.star | 95 ++++++++++++-------------------------- 1 file changed, 29 insertions(+), 66 deletions(-) diff --git a/local-deployment/main.star b/local-deployment/main.star index 7f53db617f..5b744121da 100644 --- a/local-deployment/main.star +++ b/local-deployment/main.star @@ -16,40 +16,35 @@ LITTLE_BIGTABLE_PORT_ID = "littlebigtable" EXPLORER_CONFIG_FILENAME = "config.yml" def run(plan, args): - # Add a Postgres server - postgres = plan.add_service( - name = "postgres", - config = ServiceConfig( - image = "postgres:15.2-alpine", - ports = { - POSTGRES_PORT_ID: PortSpec(5432, application_protocol = "postgresql"), - }, - env_vars = { - "POSTGRES_DB": POSTGRES_DB, - "POSTGRES_USER": POSTGRES_USER, - "POSTGRES_PASSWORD": POSTGRES_PASSWORD, - }, - ), - ) - # Add a redis server - redis = plan.add_service( - name = "redis", - config = ServiceConfig( - image = "redis:7", - ports = { - REDIS_PORT_ID: PortSpec(6379, application_protocol = "tcp"), - }, - ), - ) - # Add a little bigtable server - littlebigtable = plan.add_service( - name = "littlebigtable", - config = ServiceConfig( - image = "gobitfly/little_bigtable:latest", - ports = { - LITTLE_BIGTABLE_PORT_ID: PortSpec(9000, application_protocol = "tcp"), - }, - ), + db_services = plan.add_services( + configs={ + # Add a Postgres server + "postgres": ServiceConfig( + image = "postgres:15.2-alpine", + ports = { + POSTGRES_PORT_ID: PortSpec(5432, application_protocol = "postgresql"), + }, + env_vars = { + "POSTGRES_DB": POSTGRES_DB, + "POSTGRES_USER": POSTGRES_USER, + "POSTGRES_PASSWORD": POSTGRES_PASSWORD, + }, + ), + # Add a Redis server + "redis": ServiceConfig( + image = "redis:7", + ports = { + REDIS_PORT_ID: PortSpec(6379, application_protocol = "tcp"), + }, + ), + # Add a Bigtable Emulator server + "littlebigtable": ServiceConfig( + image = "gobitfly/little_bigtable:latest", + ports = { + LITTLE_BIGTABLE_PORT_ID: PortSpec(9000, application_protocol = "tcp"), + }, + ), + } ) # Spin up a local ethereum testnet @@ -65,38 +60,6 @@ def run(plan, args): # transaction_spammer.launch_transaction_spammer(plan, genesis_constants.PRE_FUNDED_ACCOUNTS, all_el_client_contexts[0]) # plan.print("Succesfully launched transaction spammer") - # Still WIP: - # el_uri = "http://{0}:{1}".format(all_el_client_contexts[0].ip_addr, all_el_client_contexts[0].rpc_port_num) - # redis_uri = "{0}:{1}".format(redis.ip_address, 6379) - - # plan.print("{0}".format(all_cl_client_contexts[0].ip_addr)) - - # config_template = read_file("./explorer-config-template.yml") - # template_data = new_config_template_data(all_cl_client_contexts[0], el_uri, littlebigtable.ip_address, 9000, postgres.ip_address, 5432, redis_uri) - # template_and_data = shared_utils.new_template_and_data(config_template, template_data) - # template_and_data_by_rel_dest_filepath = {} - # template_and_data_by_rel_dest_filepath[EXPLORER_CONFIG_FILENAME] = template_and_data - - # config_files_artifact_name = plan.render_templates(template_and_data_by_rel_dest_filepath, "config.yml") - - # createdbschema = plan.add_service( - # name = "createdbschema", - # config = ServiceConfig( - # image = "explorer", - # files = { - # "/app/config/": config_files_artifact_name, - # }, - # entrypoint = [ - # "./misc" - # ], - # cmd = [ - # "-config", - # "/app/config/config.yml", - # "-command", - # "applyDbSchema" - # ], - # ), - # ) def new_config_template_data(cl_node_info, el_uri, lbt_host, lbt_port, db_host, db_port, redis_uri): return { From 87aaee50c456e5cf2fb26313e9afefe467aa29e7 Mon Sep 17 00:00:00 2001 From: peter <1674920+peterbitfly@users.noreply.github.com> Date: Tue, 5 Sep 2023 09:44:02 +0200 Subject: [PATCH 074/106] (BIDS-2379) make starting the tx spammer configurable --- local-deployment/main.star | 8 +++++--- local-deployment/network-params.json | 3 ++- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/local-deployment/main.star b/local-deployment/main.star index 5b744121da..ad39062bcd 100644 --- a/local-deployment/main.star +++ b/local-deployment/main.star @@ -56,9 +56,11 @@ def run(plan, args): all_el_client_contexts.append(participant.el_client_context) all_cl_client_contexts.append(participant.cl_client_context) - # plan.print("Launching transaction spammer") - # transaction_spammer.launch_transaction_spammer(plan, genesis_constants.PRE_FUNDED_ACCOUNTS, all_el_client_contexts[0]) - # plan.print("Succesfully launched transaction spammer") + + if args["start_tx_spammer"]: + plan.print("Launching transaction spammer") + transaction_spammer.launch_transaction_spammer(plan, genesis_constants.PRE_FUNDED_ACCOUNTS, all_el_client_contexts[0]) + plan.print("Succesfully launched transaction spammer") def new_config_template_data(cl_node_info, el_uri, lbt_host, lbt_port, db_host, db_port, redis_uri): diff --git a/local-deployment/network-params.json b/local-deployment/network-params.json index 2c7231f300..e63a9cd7f4 100644 --- a/local-deployment/network-params.json +++ b/local-deployment/network-params.json @@ -24,6 +24,7 @@ "capella_fork_epoch": 5, "deneb_fork_epoch": 500 }, - "global_client_log_level": "info" + "global_client_log_level": "info", + "start_tx_spammer": true } \ No newline at end of file From 54626c7d69ace8a0fbc16e53f5013c34e6ed9735 Mon Sep 17 00:00:00 2001 From: peter <1674920+peterbitfly@users.noreply.github.com> Date: Tue, 5 Sep 2023 14:04:30 +0200 Subject: [PATCH 075/106] (BIDS-2379) fix statistics --- cmd/statistics/main.go | 2 ++ db/bigtable.go | 8 ++++++-- db/statistics.go | 9 ++++++++- exporter/sync_committees.go | 4 ++-- 4 files changed, 18 insertions(+), 5 deletions(-) diff --git a/cmd/statistics/main.go b/cmd/statistics/main.go index bcbf2f710a..8cd4f7ac09 100644 --- a/cmd/statistics/main.go +++ b/cmd/statistics/main.go @@ -237,6 +237,8 @@ func statisticsLoop(concurrencyTotal uint64, concurrencyCl uint64, concurrencyFa currentDay := latestEpoch / epochsPerDay previousDay := currentDay - 1 + logrus.Info(currentDay) + logrus.Info(previousDay) if previousDay > currentDay { previousDay = currentDay } diff --git a/db/bigtable.go b/db/bigtable.go index e1613bfe20..8a6b66804b 100644 --- a/db/bigtable.go +++ b/db/bigtable.go @@ -604,6 +604,9 @@ func (bigtable *Bigtable) SaveSyncCommitteesAssignments(startSlot, endSlot uint6 muts := make([]*gcp_bigtable.Mutation, 0, MAX_BATCH_MUTATIONS) keys := make([]string, 0, MAX_BATCH_MUTATIONS) + totalMutations := (endSlot - startSlot) * uint64(len(validators)) + processedMutations := 0 + period := startSlot / utils.Config.Chain.Config.EpochsPerSyncCommitteePeriod / utils.Config.Chain.Config.SlotsPerEpoch for i := startSlot; i <= endSlot; i++ { for _, validator := range validators { mut := gcp_bigtable.NewMutation() @@ -613,9 +616,10 @@ func (bigtable *Bigtable) SaveSyncCommitteesAssignments(startSlot, endSlot uint6 muts = append(muts, mut) keys = append(keys, key) + processedMutations++ if len(muts) == MAX_BATCH_MUTATIONS { - logger.Infof("saving %v mutations for sync duties", len(muts)) + logger.Infof("saving %v mutations for sync duties (%d / %d, period: %d)", len(muts), processedMutations, totalMutations, period) errs, err := bigtable.tableValidatorSyncCommittees.ApplyBulk(ctx, keys, muts) if err != nil { @@ -636,7 +640,7 @@ func (bigtable *Bigtable) SaveSyncCommitteesAssignments(startSlot, endSlot uint6 } if len(muts) > 0 { - logger.Infof("saving %v mutations for sync duties", len(muts)) + logger.Infof("saving %v mutations for sync duties (%d / %d, period: %v)", len(muts), processedMutations, totalMutations, period) errs, err := bigtable.tableValidatorSyncCommittees.ApplyBulk(ctx, keys, muts) if err != nil { diff --git a/db/statistics.go b/db/statistics.go index f9055a4b42..0e3b37143d 100644 --- a/db/statistics.go +++ b/db/statistics.go @@ -72,15 +72,22 @@ func WriteValidatorStatisticsForDay(day uint64, concurrencyTotal uint64, concurr } logger.Infof("getting exported state took %v", time.Since(start)) - validators, err := GetValidatorIndices() + maxValidatorIndex, err := BigtableClient.GetMaxValidatorindexForEpoch(lastEpoch) if err != nil { return err } + validators := make([]uint64, 0, maxValidatorIndex) + + logger.Info(maxValidatorIndex) + for i := uint64(0); i <= maxValidatorIndex; i++ { + validators = append(validators, i) + } if exported.FailedAttestations && exported.SyncDuties && exported.WithdrawalsDeposits && exported.Balance && exported.ClRewards && exported.ElRewards && exported.TotalAccumulation && exported.TotalPerformance && exported.BlockStats && exported.Status { logger.Infof("Skipping day %v as it is already exported", day) return nil } + logger.Info("OK") if exported.FailedAttestations { logger.Infof("Skipping failed attestations") diff --git a/exporter/sync_committees.go b/exporter/sync_committees.go index 258507d73d..dcb59fe725 100644 --- a/exporter/sync_committees.go +++ b/exporter/sync_committees.go @@ -86,13 +86,13 @@ func exportSyncCommitteeAtPeriod(rpcClient rpc.Client, p uint64) error { validatorsU64[i] = idxU64 } - dedupMap := make(map[uint64]bool, len(validatorsU64)) + dedupMap := make(map[uint64]bool) for _, validator := range validatorsU64 { dedupMap[validator] = true } - validatorsU64 = make([]uint64, len(dedupMap)) + validatorsU64 = make([]uint64, 0, len(dedupMap)) for validator := range dedupMap { validatorsU64 = append(validatorsU64, validator) } From ef0180fc0f0250c8c83a06986f59a5affa18e5c3 Mon Sep 17 00:00:00 2001 From: peter <1674920+peterbitfly@users.noreply.github.com> Date: Wed, 6 Sep 2023 12:24:06 +0200 Subject: [PATCH 076/106] (BIDS-2379) rework exporter to skip exporting advance duties, reduce write overhead during epoch export, refractor validator sync table --- db/bigtable.go | 108 ++++++++++++--- exporter/exporter.go | 16 +-- exporter/sync_committees.go | 22 ++-- handlers/validator.go | 190 ++++++++++++--------------- local-deployment/network-params.json | 2 +- rpc/lighthouse.go | 19 +++ types/exporter.go | 42 +++--- utils/utils.go | 2 +- 8 files changed, 235 insertions(+), 166 deletions(-) diff --git a/db/bigtable.go b/db/bigtable.go index 8a6b66804b..b7e903551b 100644 --- a/db/bigtable.go +++ b/db/bigtable.go @@ -693,17 +693,26 @@ func (bigtable *Bigtable) SaveAttestations(blocks map[uint64]map[string]*types.B for _, slot := range slots { for _, b := range blocks[slot] { - // logger.Infof("processing slot %v", slot) - for _, a := range b.Attestations { - for _, validator := range a.Attesters { - inclusionSlot := slot - attestedSlot := a.Data.Slot - if attestationsBySlot[attestedSlot] == nil { - attestationsBySlot[attestedSlot] = make(map[uint64]uint64) - } + if b.Status == 2 { + if attestationsBySlot[slot] == nil { + attestationsBySlot[slot] = make(map[uint64]uint64) + } + for _, validator := range b.EmptySlotAttestationAssignments { + attestationsBySlot[slot][validator] = MAX_BLOCK_NUMBER // will result in the ts of the set bigtable cell being 0 + } + } else { + // logger.Infof("processing slot %v", slot) + for _, a := range b.Attestations { + for _, validator := range a.Attesters { + inclusionSlot := slot + attestedSlot := a.Data.Slot + if attestationsBySlot[attestedSlot] == nil { + attestationsBySlot[attestedSlot] = make(map[uint64]uint64) + } - if attestationsBySlot[attestedSlot][validator] == 0 || inclusionSlot < attestationsBySlot[attestedSlot][validator] { - attestationsBySlot[attestedSlot][validator] = inclusionSlot + if attestationsBySlot[attestedSlot][validator] == 0 || inclusionSlot < attestationsBySlot[attestedSlot][validator] { + attestationsBySlot[attestedSlot][validator] = inclusionSlot + } } } } @@ -854,7 +863,13 @@ func (bigtable *Bigtable) SaveSyncComitteeDuties(blocks map[uint64]map[string]*t for _, slot := range slots { for _, b := range blocks[slot] { if b.Status == 2 { - continue + + if dutiesBySlot[b.Slot] == nil { + dutiesBySlot[b.Slot] = make(map[uint64]bool) + } + for _, validator := range b.EmptySlotSyncAssignments { + dutiesBySlot[b.Slot][validator] = false + } } else if b.SyncAggregate != nil && len(b.SyncAggregate.SyncCommitteeValidators) > 0 { bitLen := len(b.SyncAggregate.SyncCommitteeBits) * 8 valLen := len(b.SyncAggregate.SyncCommitteeValidators) @@ -1217,6 +1232,39 @@ func (bigtable *Bigtable) GetSyncParticipationBySlot(slot uint64) (uint64, error return 0, nil } +func (bigtable *Bigtable) GetSyncParticipationBySlotRange(startSlot, endSlot uint64) (map[uint64]uint64, error) { + + ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(time.Minute*5)) + defer cancel() + + startKey := fmt.Sprintf("%s:%s:%s", bigtable.chainId, SYNC_COMMITTEES_PARTICIPATION_FAMILY, bigtable.reversedPaddedSlot(endSlot)) + endKey := fmt.Sprintf("%s:%s:%s\x00", bigtable.chainId, SYNC_COMMITTEES_PARTICIPATION_FAMILY, bigtable.reversedPaddedSlot(startSlot)) + rowRange := gcp_bigtable.NewRange(startKey, endKey) + + res := make(map[uint64]uint64) + err := bigtable.tableValidatorSyncCommittees.ReadRows(ctx, rowRange, func(r gcp_bigtable.Row) bool { + keySplit := strings.Split(r.Key(), ":") + + slot, err := strconv.ParseUint(keySplit[2], 10, 64) + if err != nil { + logger.Errorf("error parsing epoch from row key %v: %v", r.Key(), err) + return false + } + slot = MAX_BLOCK_NUMBER - slot + + for _, ri := range r[SYNC_COMMITTEES_PARTICIPATION_FAMILY] { + res[slot] = binary.LittleEndian.Uint64(ri.Value) + } + + return true + }) + if err != nil { + return nil, err + } + + return res, nil +} + func (bigtable *Bigtable) GetValidatorMissedAttestationHistory(validators []uint64, startEpoch uint64, endEpoch uint64) (map[uint64]map[uint64]bool, error) { if len(validators) == 0 { @@ -1320,7 +1368,7 @@ func (bigtable *Bigtable) GetValidatorMissedAttestationHistory(validators []uint return res, nil } -func (bigtable *Bigtable) GetValidatorSyncDutiesHistory(validators []uint64, startEpoch uint64, endEpoch uint64) (map[uint64][]*types.ValidatorSyncParticipation, error) { +func (bigtable *Bigtable) GetValidatorSyncDutiesHistory(validators []uint64, startSlot uint64, endSlot uint64) (map[uint64]map[uint64]*types.ValidatorSyncParticipation, error) { if len(validators) == 0 { return nil, fmt.Errorf("passing empty validator array is unsupported") @@ -1332,7 +1380,7 @@ func (bigtable *Bigtable) GetValidatorSyncDutiesHistory(validators []uint64, sta ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(time.Minute*5)) defer cancel() - res := make(map[uint64][]*types.ValidatorSyncParticipation, len(validators)) + res := make(map[uint64]map[uint64]*types.ValidatorSyncParticipation, len(validators)) resMux := &sync.Mutex{} filter := gcp_bigtable.LatestNFilter(1) @@ -1354,7 +1402,7 @@ func (bigtable *Bigtable) GetValidatorSyncDutiesHistory(validators []uint64, sta return gCtx.Err() default: } - ranges := bigtable.getValidatorsEpochSlotRanges(vals, startEpoch, endEpoch) + ranges := bigtable.getValidatorSlotRanges(vals, startSlot, endSlot) err := bigtable.tableValidatorSyncCommittees.ReadRows(ctx, ranges, func(r gcp_bigtable.Row) bool { @@ -1383,16 +1431,16 @@ func (bigtable *Bigtable) GetValidatorSyncDutiesHistory(validators []uint64, sta resMux.Lock() if res[validator] == nil { - res[validator] = make([]*types.ValidatorSyncParticipation, 0) + res[validator] = make(map[uint64]*types.ValidatorSyncParticipation, 0) } - if len(res[validator]) > 0 && res[validator][len(res[validator])-1].Slot == slot { - res[validator][len(res[validator])-1].Status = status + if len(res[validator]) > 0 && res[validator][slot] != nil { + res[validator][slot].Status = status } else { - res[validator] = append(res[validator], &types.ValidatorSyncParticipation{ + res[validator][slot] = &types.ValidatorSyncParticipation{ Slot: slot, Status: status, - }) + } } resMux.Unlock() @@ -2293,6 +2341,28 @@ func (bigtable *Bigtable) getValidatorsEpochSlotRanges(validatorIndices []uint64 return ranges } +func (bigtable *Bigtable) getValidatorSlotRanges(validatorIndices []uint64, startSlot uint64, endSlot uint64) gcp_bigtable.RowRangeList { + + if endSlot < startSlot { // handle overflows + startSlot = 0 + } + + startEpoch := utils.EpochOfSlot(startSlot) + endEpoch := utils.EpochOfSlot(endSlot) + + ranges := make(gcp_bigtable.RowRangeList, 0, len(validatorIndices)) + + for _, validatorIndex := range validatorIndices { + validatorKey := bigtable.validatorIndexToKey(validatorIndex) + + rangeEnd := fmt.Sprintf("%s:%s:%s:%s%s", bigtable.chainId, validatorKey, bigtable.reversedPaddedEpoch(startEpoch), bigtable.reversedPaddedSlot(startSlot), "\x00") + rangeStart := fmt.Sprintf("%s:%s:%s:%s", bigtable.chainId, validatorKey, bigtable.reversedPaddedEpoch(endEpoch), bigtable.reversedPaddedSlot(endSlot)) + ranges = append(ranges, gcp_bigtable.NewRange(rangeStart, rangeEnd)) + + } + return ranges +} + func (bigtable *Bigtable) validatorIndexToKey(index uint64) string { return utils.ReverseString(fmt.Sprintf("%d", index)) } diff --git a/exporter/exporter.go b/exporter/exporter.go index dc7cf20c4d..f0c57e5d38 100644 --- a/exporter/exporter.go +++ b/exporter/exporter.go @@ -553,7 +553,7 @@ func ExportEpoch(epoch uint64, client rpc.Client) error { // export epoch data to bigtable g := new(errgroup.Group) - g.SetLimit(7) + g.SetLimit(5) g.Go(func() error { err = db.BigtableClient.SaveValidatorBalances(epoch, data.Validators) if err != nil { @@ -561,13 +561,13 @@ func ExportEpoch(epoch uint64, client rpc.Client) error { } return nil }) - g.Go(func() error { - err = db.BigtableClient.SaveAttestationAssignments(epoch, data.ValidatorAssignmentes.AttestorAssignments) - if err != nil { - return fmt.Errorf("error exporting attestation assignments to bigtable: %v", err) - } - return nil - }) + // g.Go(func() error { + // err = db.BigtableClient.SaveAttestationAssignments(epoch, data.ValidatorAssignmentes.AttestorAssignments) + // if err != nil { + // return fmt.Errorf("error exporting attestation assignments to bigtable: %v", err) + // } + // return nil + // }) g.Go(func() error { err = db.BigtableClient.SaveProposalAssignments(epoch, data.ValidatorAssignmentes.ProposerAssignments) if err != nil { diff --git a/exporter/sync_committees.go b/exporter/sync_committees.go index dcb59fe725..05159371ce 100644 --- a/exporter/sync_committees.go +++ b/exporter/sync_committees.go @@ -97,17 +97,17 @@ func exportSyncCommitteeAtPeriod(rpcClient rpc.Client, p uint64) error { validatorsU64 = append(validatorsU64, validator) } - start := time.Now() - firstSlot := firstEpoch * utils.Config.Chain.Config.SlotsPerEpoch - lastSlot := lastEpoch*utils.Config.Chain.Config.SlotsPerEpoch + utils.Config.Chain.Config.SlotsPerEpoch - 1 - logger.Infof("exporting sync committee assignments for period %v (epoch %v to %v, slot %v to %v) to bigtable", p, firstEpoch, lastEpoch, firstSlot, lastSlot) - - err = db.BigtableClient.SaveSyncCommitteesAssignments(firstSlot, lastSlot, validatorsU64) - if err != nil { - return fmt.Errorf("error saving sync committee assignments: %v", err) - } - logger.Infof("exported sync committee assignments for period %v to bigtable in %v", p, time.Since(start)) - + // start := time.Now() + // + // firstSlot := firstEpoch * utils.Config.Chain.Config.SlotsPerEpoch + // lastSlot := lastEpoch*utils.Config.Chain.Config.SlotsPerEpoch + utils.Config.Chain.Config.SlotsPerEpoch - 1 + // logger.Infof("exporting sync committee assignments for period %v (epoch %v to %v, slot %v to %v) to bigtable", p, firstEpoch, lastEpoch, firstSlot, lastSlot) + + // err = db.BigtableClient.SaveSyncCommitteesAssignments(firstSlot, lastSlot, validatorsU64) + // if err != nil { + // return fmt.Errorf("error saving sync committee assignments: %v", err) + // } + // logger.Infof("exported sync committee assignments for period %v to bigtable in %v", p, time.Since(start)) tx, err := db.WriterDb.Beginx() if err != nil { return err diff --git a/handlers/validator.go b/handlers/validator.go index b48b821cc0..4764d61956 100644 --- a/handlers/validator.go +++ b/handlers/validator.go @@ -2,7 +2,6 @@ package handlers import ( "bytes" - "context" "database/sql" "encoding/hex" "encoding/json" @@ -19,7 +18,6 @@ import ( "sort" "strconv" "strings" - "sync" "time" "github.com/ethereum/go-ethereum/accounts" @@ -77,6 +75,7 @@ func Validator(w http.ResponseWriter, r *http.Request) { var err error latestEpoch := services.LatestEpoch() + latestProposedSlot := services.LatestProposedSlot() lastFinalizedEpoch := services.LatestFinalizedEpoch() validatorPageData := types.ValidatorPageData{} @@ -299,7 +298,7 @@ func Validator(w http.ResponseWriter, r *http.Request) { } validatorPageData.LastAttestationSlot = lastAttestationSlots[index] - lastStatsDay, _ := services.LatestExportedStatisticDay() + lastStatsDay, lastStatsDayErr := services.LatestExportedStatisticDay() timings.BasicInfo = time.Since(timings.Start) @@ -697,9 +696,13 @@ func Validator(w http.ResponseWriter, r *http.Request) { // if sync duties of last period haven't fully been exported yet, fetch remaining duties from bigtable lastExportedEpoch := (lastStatsDay+1)*utils.EpochsPerDay() - 1 + + if lastStatsDayErr == db.ErrNoStats { + lastExportedEpoch = 0 + } lastSyncPeriod := actualSyncPeriods[0] if lastSyncPeriod.LastEpoch > lastExportedEpoch { - res, err := db.BigtableClient.GetValidatorSyncDutiesHistory([]uint64{index}, lastExportedEpoch+1, latestEpoch) + res, err := db.BigtableClient.GetValidatorSyncDutiesHistory([]uint64{index}, lastExportedEpoch+1*utils.Config.Chain.Config.SlotsPerEpoch, latestProposedSlot) if err != nil { return fmt.Errorf("error retrieving validator sync participations data from bigtable: %v", err) } @@ -1834,8 +1837,6 @@ func ValidatorStatsTable(w http.ResponseWriter, r *http.Request) { // ValidatorSync retrieves one page of sync duties of a specific validator for DataTable. func ValidatorSync(w http.ResponseWriter, r *http.Request) { - ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(time.Minute*10)) - defer cancel() w.Header().Set("Content-Type", "application/json") vars := mux.Vars(r) @@ -1878,126 +1879,80 @@ func ValidatorSync(w http.ResponseWriter, r *http.Request) { SELECT period FROM sync_committees WHERE validatorindex = $1 - ORDER BY period asc`, validatorIndex) + ORDER BY period desc`, validatorIndex) if err != nil { logger.WithError(err).Errorf("error getting sync tab count data of sync-assignments") http.Error(w, "Internal server error", http.StatusInternalServerError) return } - latestEpoch := services.LatestEpoch() - - var syncSlots []uint64 = []uint64{} - // getting all sync slots until the latestEpoch (so we exclude scheduled) - for _, syncPeriod := range syncPeriods { - startEpoch := syncPeriod * utils.Config.Chain.Config.EpochsPerSyncCommitteePeriod - endEpoch := (syncPeriod + 1) * utils.Config.Chain.Config.EpochsPerSyncCommitteePeriod - for epoch := startEpoch; epoch < endEpoch; epoch++ { - if epoch <= latestEpoch { - for slot := epoch * utils.Config.Chain.Config.SlotsPerEpoch; slot < (epoch+1)*utils.Config.Chain.Config.SlotsPerEpoch; slot++ { - syncSlots = append(syncSlots, slot) - } - } - } - } - // total count of sync duties for this validator - totalCount := uint64(len(syncSlots)) + totalCount := uint64(0) tableData := [][]interface{}{} - if totalCount > 0 && start <= totalCount { + if len(syncPeriods) > 0 { - // if ordering is desc, reverse sync slots - if descOrdering { - utils.ReverseSlice(syncSlots) - } - - // let's pick the slots we want to display on the requested page - pageSlots := syncSlots[start:protomath.MinU64(start+length, totalCount)] - length = uint64(len(pageSlots)) // Last page might be shorter - epochs := []uint64{} - last := uint64(0) - // we gather all epochs for that page (the table currently displays 10 items per page, so it displays max 2 epoches. But we want to be future proof, so we support bigger page sizes) - for _, slot := range pageSlots { - epoch := slot / utils.Config.Chain.Config.SlotsPerEpoch - if len(epochs) == 0 || epoch != last { - epochs = append(epochs, epoch) - last = epoch + latestProposedSlot := services.LatestProposedSlot() + + slots := make([]uint64, 0, utils.Config.Chain.Config.EpochsPerSyncCommitteePeriod*utils.Config.Chain.Config.SlotsPerEpoch*uint64(len(syncPeriods))) + + for _, period := range syncPeriods { + firstEpoch := utils.FirstEpochOfSyncPeriod(period) + lastEpoch := firstEpoch + utils.Config.Chain.Config.EpochsPerSyncCommitteePeriod - 1 + + firstSlot := firstEpoch * utils.Config.Chain.Config.SlotsPerEpoch + lastSlot := lastEpoch*utils.Config.Chain.Config.SlotsPerEpoch - 1 + + for slot := firstSlot; slot <= lastSlot; slot++ { + if slot > latestProposedSlot { + continue + } + slots = append(slots, slot) } } - type PageSlot struct { - Status uint64 - Participation uint64 - } - pageSlotsMap := make(map[uint64]PageSlot) - for i := 0; i < len(pageSlots); i++ { - pageSlotsMap[pageSlots[i]] = PageSlot{} + // if ordering is desc, reverse sync slots + if descOrdering { + utils.ReverseSlice(slots) } - var missedSyncSlots []uint64 + totalCount = uint64(len(slots)) - mux := sync.Mutex{} - g, gCtx := errgroup.WithContext(ctx) + startIndex := start + length - 1 + if startIndex > uint64(len(slots)-1) { + startIndex = uint64(len(slots) - 1) + } + endIndex := start - // load the sync duties for all epochs involved from bt - for i := 0; i < len(epochs); i++ { - epoch := epochs[i] - g.Go(func() error { - select { - case <-gCtx.Done(): - return gCtx.Err() - default: - } + startSlot := slots[startIndex] - syncDuties, err := db.BigtableClient.GetValidatorSyncDutiesHistory([]uint64{validatorIndex}, epoch, epoch) - if err != nil { - return fmt.Errorf("error retrieving validator [%v] sync duty data from bigtable for epoch [%v]: %w", validatorIndex, epoch, err) - } else if uint64(len(syncDuties[validatorIndex]))%utils.Config.Chain.Config.SlotsPerEpoch > 0 { - return fmt.Errorf("wrong number [%v] of syncDuties for validator [%v] received from from bigtable for epoch [%v]", len(syncDuties[validatorIndex]), validatorIndex, epoch) - } + endSlot := slots[endIndex] - for _, duties := range syncDuties { - for _, duty := range duties { - mux.Lock() - pageSlot, ok := pageSlotsMap[duty.Slot] - if !ok { - mux.Unlock() - continue // this is not the slot we are looking for - } - - participation, err := db.BigtableClient.GetSyncParticipationBySlot(duty.Slot) - if err != nil { - return fmt.Errorf("error retrieving sync participation for slot %v", duty.Slot) - } - pageSlot.Participation = participation - - slotTime := utils.SlotToTime(duty.Slot) - if duty.Status == 0 && time.Since(slotTime) <= time.Minute { - duty.Status = 2 // scheduled - } - - if duty.Status == 0 { - missedSyncSlots = append(missedSyncSlots, duty.Slot) - } - pageSlot.Status = duty.Status - pageSlotsMap[duty.Slot] = pageSlot - mux.Unlock() - } - } - return nil - }) + if startSlot > endSlot { // guard against overflow + startSlot = 0 } - if err := g.Wait(); err != nil { - utils.LogError(err, "", 0) + syncDuties, err := db.BigtableClient.GetValidatorSyncDutiesHistory([]uint64{validatorIndex}, startSlot, endSlot) + + if err != nil { + utils.LogError(fmt.Errorf("error retrieving validator [%v] sync duty data from bigtable for slots [%v-%v]: %w", validatorIndex, startSlot, endSlot, err), "", 0) http.Error(w, "Internal server error", http.StatusInternalServerError) return } + // spew.Dump(syncDuties[validatorIndex]) // Search for the missed slots (status = 2), to see if it was only our validator that missed the slot or if the block was missed + slotsRange := slots[endIndex : startIndex+1] + + participations, err := db.BigtableClient.GetSyncParticipationBySlotRange(startSlot, endSlot) + if err != nil { + utils.LogError(fmt.Errorf("error retrieving validator [%v] sync participation data from bigtable for slots [%v-%v]: %w", validatorIndex, startSlot, endSlot, err), "", 0) + http.Error(w, "Internal server error", http.StatusInternalServerError) + return + } + missedSlots := []uint64{} - err = db.ReaderDb.Select(&missedSlots, `SELECT slot FROM blocks WHERE slot = ANY($1) AND status = '2'`, missedSyncSlots) + err = db.ReaderDb.Select(&missedSlots, `SELECT slot FROM blocks WHERE slot = ANY($1) AND status = '2'`, slotsRange) if err != nil { logger.WithError(err).Errorf("error getting missed slots data") http.Error(w, "Internal server error", http.StatusInternalServerError) @@ -2009,24 +1964,47 @@ func ValidatorSync(w http.ResponseWriter, r *http.Request) { } // extract correct slots - tableData = make([][]interface{}, length) - for i := 0; i < len(pageSlots); i++ { - slot := pageSlots[i] - pageSlot := pageSlotsMap[slot] - participation := pageSlot.Participation + tableData = make([][]interface{}, 0, length) + for slot := endSlot; slot >= startSlot; slot-- { epoch := utils.EpochOfSlot(slot) - status := pageSlot.Status + participation := participations[slot] + + if syncDuties[validatorIndex] == nil { + tableData = append(tableData, + []interface{}{ + fmt.Sprintf("%d", utils.SyncPeriodOfEpoch(epoch)), + utils.FormatEpoch(epoch), + utils.FormatBlockSlot(slot), + utils.FormatSyncParticipationStatus(0, slot), + utils.FormatSyncParticipations(participation), + }, + ) + } + + // if utils.SlotToTime(slot).Before(time.Now()) { + // continue + // } + + status := uint64(0) + + if syncDuties[validatorIndex][slot] != nil { + status = syncDuties[validatorIndex][slot].Status + } if _, ok := missedSlotsMap[slot]; ok { status = 3 } - tableData[i] = []interface{}{ + tableData = append(tableData, []interface{}{ fmt.Sprintf("%d", utils.SyncPeriodOfEpoch(epoch)), utils.FormatEpoch(epoch), utils.FormatBlockSlot(slot), utils.FormatSyncParticipationStatus(status, slot), utils.FormatSyncParticipations(participation), + }) + + if slot == 0 { + break } } } diff --git a/local-deployment/network-params.json b/local-deployment/network-params.json index e63a9cd7f4..9900af36a0 100644 --- a/local-deployment/network-params.json +++ b/local-deployment/network-params.json @@ -25,6 +25,6 @@ "deneb_fork_epoch": 500 }, "global_client_log_level": "info", - "start_tx_spammer": true + "start_tx_spammer": false } \ No newline at end of file diff --git a/rpc/lighthouse.go b/rpc/lighthouse.go index 0775bd2e9e..b05789ad30 100644 --- a/rpc/lighthouse.go +++ b/rpc/lighthouse.go @@ -400,6 +400,25 @@ func (lc *LighthouseClient) GetEpochData(epoch uint64, skipHistoricBalances bool // Block is in the past, set status to missed data.Blocks[slot]["0x0"].Status = 2 data.Blocks[slot]["0x0"].BlockRoot = []byte{0x1} + + logger.Infof("backfilling data for missed slot %v", slot) + // Fill sync & attestation assignments + data.Blocks[slot]["0x0"].EmptySlotSyncAssignments = data.ValidatorAssignmentes.SyncAssignments + data.Blocks[slot]["0x0"].EmptySlotAttestationAssignments = make([]uint64, 0, len(data.Validators)/int(utils.Config.Chain.Config.SlotsPerEpoch)) + for key, validator := range data.ValidatorAssignmentes.AttestorAssignments { + keySplit := strings.Split(key, "-") + if len(keySplit) != 3 { + return nil, fmt.Errorf("error parsing slot from validator attestation assignment key %s: invalid format", key) + } + attestedSlot, err := strconv.ParseUint(keySplit[0], 10, 64) + if err != nil { + return nil, fmt.Errorf("error parsing slot from validator attestation assignment key %s: %w", key, err) + } + + if slot == attestedSlot { + data.Blocks[slot]["0x0"].EmptySlotAttestationAssignments = append(data.Blocks[slot]["0x0"].EmptySlotAttestationAssignments, validator) + } + } } } } diff --git a/types/exporter.go b/types/exporter.go index bc6ba2ee3b..c45fad2013 100644 --- a/types/exporter.go +++ b/types/exporter.go @@ -103,26 +103,28 @@ type SyncAggregate struct { // Block is a struct to hold block data type Block struct { - Status uint64 - Proposer uint64 - BlockRoot []byte - Slot uint64 - ParentRoot []byte - StateRoot []byte - Signature []byte - RandaoReveal []byte - Graffiti []byte - Eth1Data *Eth1Data - BodyRoot []byte - ProposerSlashings []*ProposerSlashing - AttesterSlashings []*AttesterSlashing - Attestations []*Attestation - Deposits []*Deposit - VoluntaryExits []*VoluntaryExit - SyncAggregate *SyncAggregate // warning: sync aggregate may be nil, for phase0 blocks - ExecutionPayload *ExecutionPayload // warning: payload may be nil, for phase0/altair blocks - Canonical bool - SignedBLSToExecutionChange []*SignedBLSToExecutionChange + Status uint64 + Proposer uint64 + BlockRoot []byte + Slot uint64 + ParentRoot []byte + StateRoot []byte + Signature []byte + RandaoReveal []byte + Graffiti []byte + Eth1Data *Eth1Data + BodyRoot []byte + ProposerSlashings []*ProposerSlashing + AttesterSlashings []*AttesterSlashing + Attestations []*Attestation + Deposits []*Deposit + VoluntaryExits []*VoluntaryExit + SyncAggregate *SyncAggregate // warning: sync aggregate may be nil, for phase0 blocks + ExecutionPayload *ExecutionPayload // warning: payload may be nil, for phase0/altair blocks + Canonical bool + SignedBLSToExecutionChange []*SignedBLSToExecutionChange + EmptySlotAttestationAssignments []uint64 + EmptySlotSyncAssignments []uint64 } type SignedBLSToExecutionChange struct { diff --git a/utils/utils.go b/utils/utils.go index 87e165b944..ac2ebb54cc 100644 --- a/utils/utils.go +++ b/utils/utils.go @@ -1408,7 +1408,7 @@ func GetRemainingScheduledSync(validatorCount int, stats types.SyncCommitteesSta // - `validators` : the validators to add the stats for // - `syncDutiesHistory` : the sync duties history of all queried validators // - `stats` : the stats object to add the stats to, if nil a new stats object is created -func AddSyncStats(validators []uint64, syncDutiesHistory map[uint64][]*types.ValidatorSyncParticipation, stats *types.SyncCommitteesStats) types.SyncCommitteesStats { +func AddSyncStats(validators []uint64, syncDutiesHistory map[uint64]map[uint64]*types.ValidatorSyncParticipation, stats *types.SyncCommitteesStats) types.SyncCommitteesStats { if stats == nil { stats = &types.SyncCommitteesStats{} } From 2697d97793047295a8d2223d1f2ea751bdd39496 Mon Sep 17 00:00:00 2001 From: peter <1674920+peterbitfly@users.noreply.github.com> Date: Wed, 6 Sep 2023 13:36:02 +0200 Subject: [PATCH 077/106] (BIDS-2379) add helper script to exit validators for local devnets --- local-deployment/README.md | 6 ++++++ local-deployment/exit_validator.sh | 33 ++++++++++++++++++++++++++++++ rpc/lighthouse.go | 2 +- 3 files changed, 40 insertions(+), 1 deletion(-) create mode 100644 local-deployment/exit_validator.sh diff --git a/local-deployment/README.md b/local-deployment/README.md index 68f5feb54b..1a7ca5257f 100644 --- a/local-deployment/README.md +++ b/local-deployment/README.md @@ -60,3 +60,9 @@ cd ~/eth2-beaconchain-explorer/local-deployment/ docker-compose up -d ``` You can start / stop the exporter submodules using `docker-compose` + +# Exit validators +Exiting individual validators can be done using the provided `exit_validator.sh` script. Requires [https://github.com/wealdtech/ethdo](ethdo) to be available on the path. +``` +bash exit_validators.sh -i validator_index -m "memonic" -b "http://bn_api_host:bn_api_port" +``` diff --git a/local-deployment/exit_validator.sh b/local-deployment/exit_validator.sh new file mode 100644 index 0000000000..1bb3f0f9c4 --- /dev/null +++ b/local-deployment/exit_validator.sh @@ -0,0 +1,33 @@ +#! /bin/bash +set -e + +clean_up () { + ARG=$? + rm -rf /tmp/full_withdrawal + exit $ARG +} +trap clean_up EXIT + +while getopts b:i:m: flag +do + case "${flag}" in + b) bn_endpoint=${OPTARG};; + i) index=${OPTARG};; + m) mnemonic=${OPTARG};; + esac +done +echo "Validator Index: $index"; +echo "Mnemonic: $mnemonic"; +echo "BN Endpoint: $bn_endpoint"; + +mkdir -p /tmp/full_withdrawal +echo "creating wallet" +ethdo wallet create --base-dir=/tmp/full_withdrawal --type=hd --wallet=withdrawal-validators --mnemonic="$mnemonic" --wallet-passphrase="superSecure" --allow-weak-passphrases +echo "deriving account wallet" +ethdo account create --base-dir=/tmp/full_withdrawal --account=withdrawal-validators/$index --wallet-passphrase="superSecure" --passphrase="superSecure" --allow-weak-passphrases --path="m/12381/3600/$index/0/0" +echo "creating exit message" +ethdo validator exit --base-dir=/tmp/full_withdrawal --json --account=withdrawal-validators/$index --passphrase="superSecure" --connection=$bn_endpoint > /tmp/full_withdrawal/withdrawal-$index.json +echo "submitting exit message" +ethdo validator exit --signed-operations=$(cat /tmp/full_withdrawal/withdrawal-$index.json) --connection=$bn_endpoint +ethdo wallet delete --base-dir=/tmp/full_withdrawal --wallet=withdrawal-validators +rm -rf /tmp/full_withdrawal \ No newline at end of file diff --git a/rpc/lighthouse.go b/rpc/lighthouse.go index b05789ad30..1d52ca64eb 100644 --- a/rpc/lighthouse.go +++ b/rpc/lighthouse.go @@ -177,7 +177,7 @@ func (lc *LighthouseClient) GetEpochAssignments(epoch uint64) (*types.EpochAssig proposerResp, err := lc.get(fmt.Sprintf("%s/eth/v1/validator/duties/proposer/%d", lc.endpoint, epoch)) if err != nil { - return nil, fmt.Errorf("error retrieving proposer duties: %v", err) + return nil, fmt.Errorf("error retrieving proposer duties for epoch %v: %v", epoch, err) } var parsedProposerResponse StandardProposerDutiesResponse err = json.Unmarshal(proposerResp, &parsedProposerResponse) From 7fd95f4c5505550262666babdd12c4e8ee7aa389 Mon Sep 17 00:00:00 2001 From: peter <1674920+peterbitfly@users.noreply.github.com> Date: Wed, 6 Sep 2023 14:45:59 +0200 Subject: [PATCH 078/106] (BIDS-2379) adds scripts for withdrawing, exiting & depositing validators --- go.mod | 33 +- go.sum | 508 ++++++++++++++++++ local-deployment/README.md | 8 + local-deployment/network-params.json | 4 +- local-deployment/provision-explorer-config.sh | 10 +- .../scripts/add_withdrawal_addr.sh | 37 ++ local-deployment/scripts/deposit_validator.sh | 49 ++ .../{ => scripts}/exit_validator.sh | 0 templates/deposits.html | 4 +- 9 files changed, 643 insertions(+), 10 deletions(-) create mode 100644 local-deployment/scripts/add_withdrawal_addr.sh create mode 100644 local-deployment/scripts/deposit_validator.sh rename local-deployment/{ => scripts}/exit_validator.sh (100%) diff --git a/go.mod b/go.mod index 268ba9369b..243fb9d8d4 100644 --- a/go.mod +++ b/go.mod @@ -42,7 +42,7 @@ require ( github.com/pkg/errors v0.9.1 github.com/pressly/goose/v3 v3.10.0 github.com/prometheus/client_golang v1.14.0 - github.com/protolambda/zrnt v0.12.4 + github.com/protolambda/zrnt v0.30.0 github.com/prysmaticlabs/go-bitfield v0.0.0-20210809151128-385d8c5e3fb7 github.com/prysmaticlabs/go-ssz v0.0.0-20210121151755-f6208871c388 github.com/prysmaticlabs/prysm/v3 v3.2.0 @@ -79,7 +79,13 @@ require ( cloud.google.com/go/compute/metadata v0.2.1 // indirect cloud.google.com/go/iam v0.7.0 // indirect cloud.google.com/go/longrunning v0.3.0 // indirect + github.com/FactomProject/basen v0.0.0-20150613233007-fe3947df716e // indirect + github.com/FactomProject/btcutilecc v0.0.0-20130527213604-d3a63a5752ec // indirect + github.com/FactomProject/go-bip32 v0.3.5 // indirect + github.com/FactomProject/go-bip39 v0.3.5 // indirect + github.com/FactomProject/go-bip44 v0.0.0-20190306062959-b541a96d8da9 // indirect github.com/alessio/shellescape v1.4.1 // indirect + github.com/antlr/antlr4/runtime/Go/antlr v0.0.0-20210716071054-a231a1a7f1cc // indirect github.com/carlmjohnson/requests v0.23.4 // indirect github.com/census-instrumentation/opencensus-proto v0.2.1 // indirect github.com/cncf/udpa/go v0.0.0-20210930031921-04548b0d99d4 // indirect @@ -92,8 +98,15 @@ require ( github.com/envoyproxy/protoc-gen-validate v0.1.0 // indirect github.com/fsnotify/fsnotify v1.6.0 // indirect github.com/googleapis/enterprise-certificate-proxy v0.2.0 // indirect + github.com/hashicorp/hcl v1.0.0 // indirect + github.com/holiman/uint256 v1.2.0 // indirect github.com/imdario/mergo v0.3.13 // indirect + github.com/inconshreveable/mousetrap v1.0.0 // indirect github.com/ipfs/go-cid v0.3.2 // indirect + github.com/karalabe/usb v0.0.2 // indirect + github.com/kilic/bls12-381 v0.1.0 // indirect + github.com/magiconair/properties v1.8.5 // indirect + github.com/miekg/dns v1.1.50 // indirect github.com/mitchellh/go-homedir v1.1.0 // indirect github.com/multiformats/go-base32 v0.1.0 // indirect github.com/multiformats/go-base36 v0.2.0 // indirect @@ -101,17 +114,35 @@ require ( github.com/multiformats/go-multihash v0.2.1 // indirect github.com/multiformats/go-varint v0.0.7 // indirect github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 // indirect + github.com/pborman/uuid v1.2.1 // indirect + github.com/pelletier/go-toml v1.9.3 // indirect + github.com/protolambda/bls12-381-util v0.0.0-20210720105258-a772f2aac13e // indirect + github.com/protolambda/eth2-val-tools v0.1.2-0.20230223182759-4bf01453537a // indirect + github.com/protolambda/go-keystorev4 v0.0.0-20211007151826-f20444f6d564 // indirect github.com/protolambda/zssz v0.1.5 // indirect + github.com/protolambda/ztyp v0.2.2 // indirect github.com/prysmaticlabs/fastssz v0.0.0-20221107182844-78142813af44 // indirect github.com/prysmaticlabs/gohashtree v0.0.2-alpha // indirect github.com/rs/cors v1.8.0 // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect github.com/spaolacci/murmur3 v1.1.0 // indirect + github.com/spf13/afero v1.6.0 // indirect + github.com/spf13/cast v1.4.0 // indirect + github.com/spf13/cobra v1.2.1 // indirect + github.com/spf13/jwalterweatherman v1.1.0 // indirect + github.com/spf13/pflag v1.0.5 // indirect + github.com/spf13/viper v1.8.1 // indirect + github.com/subosito/gotenv v1.2.0 // indirect + github.com/tyler-smith/go-bip39 v1.1.0 // indirect github.com/urfave/cli v1.22.12 // indirect + github.com/wealdtech/ethereal v0.0.0-20211110093529-ed2beb2b4a38 // indirect github.com/wealdtech/go-bytesutil v1.2.1 // indirect + github.com/wealdtech/go-erc1820 v1.2.3 // indirect github.com/wealdtech/go-merkletree v1.0.1-0.20190605192610-2bb163c2ea2a // indirect github.com/wealdtech/go-multicodec v1.4.0 // indirect + github.com/wealdtech/go-string2eth v1.1.0 // indirect github.com/yusufpapurcu/wmi v1.2.2 // indirect + golang.org/x/mod v0.9.0 // indirect google.golang.org/genproto v0.0.0-20221118155620-16455021b5e6 // indirect google.golang.org/grpc v1.52.3 // indirect lukechampine.com/blake3 v1.1.7 // indirect diff --git a/go.sum b/go.sum index 59ed5a5af9..b0d4ae04d2 100644 --- a/go.sum +++ b/go.sum @@ -1,11 +1,13 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= +cloud.google.com/go v0.43.0/go.mod h1:BOSR3VbTLkk6FDC/TcffxP4NF/FFBGA5ku+jvKOP7pg= cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU= cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc= cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0= cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6To= +cloud.google.com/go v0.51.0/go.mod h1:hWtGJ6gnXH+KgDv+V0zFGDvpi07n3z8ZNj3T1RW0Gcw= cloud.google.com/go v0.52.0/go.mod h1:pXajvRH/6o3+F9jDHZWQ5PbGhn+o8w9qiu/CffaVdO4= cloud.google.com/go v0.53.0/go.mod h1:fp/UouUEsRkN6ryDKNW/Upv/JBKnv6WDthjR6+vze6M= cloud.google.com/go v0.54.0/go.mod h1:1rq2OEkV3YMf6n/9ZvGWI3GWw0VoqH/1x2nd8Is/bPc= @@ -15,6 +17,9 @@ cloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOY cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHObY= cloud.google.com/go v0.72.0/go.mod h1:M+5Vjvlc2wnp6tjzE102Dw08nGShTscUx2nZMufOKPI= cloud.google.com/go v0.74.0/go.mod h1:VV1xSbzvo+9QJOxLDaJfTjx5e+MePCpCWwvftOeQmWk= +cloud.google.com/go v0.78.0/go.mod h1:QjdrLG0uq+YwhjoVOLsS1t7TW8fs36kLs4XO5R5ECHg= +cloud.google.com/go v0.79.0/go.mod h1:3bzgcEeQlzbuEAYu4mrWhKqWjmpprinYgKJLgKHnbb8= +cloud.google.com/go v0.81.0/go.mod h1:mk/AM35KwGk/Nm2YSeZbxXdrNK3KZOYHmLkOqC2V6E0= cloud.google.com/go v0.105.0 h1:DNtEKRBAAzeS4KyIory52wWHuClNaXJ5x1F7xa4q+5Y= cloud.google.com/go v0.105.0/go.mod h1:PrLgOJNe5nfE9UMxKxgXj4mD3voiP+YQ6gdt6KMFOKM= cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= @@ -23,6 +28,7 @@ cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvf cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg= cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc= cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ= +cloud.google.com/go/bigtable v1.2.0/go.mod h1:JcVAOl45lrTmQfLj7T6TxyMzIN/3FGGcFm+2xVAli2o= cloud.google.com/go/bigtable v1.16.0 h1:sqJhhslzQOag49Mf2/uH3+u+NdfpPX0gjKAcgYpRUCU= cloud.google.com/go/bigtable v1.16.0/go.mod h1:6f7WVXfeZaJz0xevUZoTA1s8sTmmrQqIAkRDVEHVg7I= cloud.google.com/go/compute v1.12.1 h1:gKVJMEyqV5c/UnpzjjQbo3Rjvvqpr9B1DFSbJC4OXr0= @@ -51,15 +57,40 @@ cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RX cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= cloud.google.com/go/storage v1.27.0 h1:YOO045NZI9RKfCj1c5A/ZtuuENUc8OAW+gHdGnDgyMQ= cloud.google.com/go/storage v1.27.0/go.mod h1:x9DOL8TK/ygDUMieqwfhdpQryTeEkhGKMi80i/iqR2s= +collectd.org v0.3.0/go.mod h1:A/8DzQBkF6abtvrT2j/AU/4tiBgJWYyh0y/oB/4MlWE= contrib.go.opencensus.io/exporter/jaeger v0.2.1 h1:yGBYzYMewVL0yO9qqJv3Z5+IRhPdU7e9o/2oKpX4YvI= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= firebase.google.com/go v3.13.0+incompatible h1:3TdYC3DDi6aHn20qoRkxwGqNgdjtblwVAyRLQwGn/+4= firebase.google.com/go v3.13.0+incompatible/go.mod h1:xlah6XbEyW6tbfSklcfe5FHJIwjt8toICdV5Wh9ptHs= +github.com/Azure/azure-pipeline-go v0.2.1/go.mod h1:UGSo8XybXnIGZ3epmeBw7Jdz+HiUVpqIlpz/HKHylF4= +github.com/Azure/azure-pipeline-go v0.2.2/go.mod h1:4rQ/NZncSvGqNkkOsNpOU1tgoNuIlp9AfUH5G1tvCHc= +github.com/Azure/azure-storage-blob-go v0.7.0/go.mod h1:f9YQKtsG1nMisotuTPpO0tjNuEjKRYAcJU8/ydDI++4= +github.com/Azure/go-autorest/autorest v0.9.0/go.mod h1:xyHB1BMZT0cuDHU7I0+g046+BFDTQ8rEZB0s4Yfa6bI= +github.com/Azure/go-autorest/autorest/adal v0.5.0/go.mod h1:8Z9fGy2MpX0PvDjB1pEgQTmVqjGhiHBW7RJJEciWzS0= +github.com/Azure/go-autorest/autorest/adal v0.8.0/go.mod h1:Z6vX6WXXuyieHAXwMj0S6HY6e6wcHn37qQMBQlvY3lc= +github.com/Azure/go-autorest/autorest/date v0.1.0/go.mod h1:plvfp3oPSKwf2DNjlBjWF/7vwR+cUD/ELuzDCXwHUVA= +github.com/Azure/go-autorest/autorest/date v0.2.0/go.mod h1:vcORJHLJEh643/Ioh9+vPmf1Ij9AEBM5FuBIXLmIy0g= +github.com/Azure/go-autorest/autorest/mocks v0.1.0/go.mod h1:OTyCOPRA2IgIlWxVYxBee2F5Gr4kF2zd2J5cFRaIDN0= +github.com/Azure/go-autorest/autorest/mocks v0.2.0/go.mod h1:OTyCOPRA2IgIlWxVYxBee2F5Gr4kF2zd2J5cFRaIDN0= +github.com/Azure/go-autorest/autorest/mocks v0.3.0/go.mod h1:a8FDP3DYzQ4RYfVAxAN3SVSiiO77gL2j2ronKKP0syM= +github.com/Azure/go-autorest/logger v0.1.0/go.mod h1:oExouG+K6PryycPJfVSxi/koC6LSNgds39diKLz7Vrc= +github.com/Azure/go-autorest/tracing v0.5.0/go.mod h1:r/s2XiOKccPW3HrqB+W0TQzfbtp2fGCgRFtBroKn4Dk= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/toml v1.2.1 h1:9F2/+DoOYIOksmaJFPw1tGFy1eDnIJXg+UHjuD8lTak= github.com/BurntSushi/toml v1.2.1/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= +github.com/DATA-DOG/go-sqlmock v1.3.3/go.mod h1:f/Ixk793poVmq4qj/V1dPUg2JEAKC73Q5eFN3EC/SaM= github.com/DataDog/zstd v1.5.2 h1:vUG4lAyuPCXO0TLbXvPv7EB7cNK1QV/luu55UHLrrn8= +github.com/FactomProject/basen v0.0.0-20150613233007-fe3947df716e h1:ahyvB3q25YnZWly5Gq1ekg6jcmWaGj/vG/MhF4aisoc= +github.com/FactomProject/basen v0.0.0-20150613233007-fe3947df716e/go.mod h1:kGUqhHd//musdITWjFvNTHn90WG9bMLBEPQZ17Cmlpw= +github.com/FactomProject/btcutilecc v0.0.0-20130527213604-d3a63a5752ec h1:1Qb69mGp/UtRPn422BH4/Y4Q3SLUrD9KHuDkm8iodFc= +github.com/FactomProject/btcutilecc v0.0.0-20130527213604-d3a63a5752ec/go.mod h1:CD8UlnlLDiqb36L110uqiP2iSflVjx9g/3U9hCI4q2U= +github.com/FactomProject/go-bip32 v0.3.5 h1:etsJ4Y/wA7spZl/WG830ogTHQivGcV/8sCrrmkQICNQ= +github.com/FactomProject/go-bip32 v0.3.5/go.mod h1:efm/M7J/CGmQ5dPtGM0GWod5LuyShuFET6oY13168w4= +github.com/FactomProject/go-bip39 v0.3.5 h1:l9g92TeqCkC5NZhm72igTpf5yaYDp3Sy4CvnPYknp6U= +github.com/FactomProject/go-bip39 v0.3.5/go.mod h1:ygPVOtW424QxnJMze9XYDeh4wT19V3iVDOqVUl/USkE= +github.com/FactomProject/go-bip44 v0.0.0-20190306062959-b541a96d8da9 h1:Wprj9FTxqhjgl7e8ZHScGwF5Wc2WrHTbEDoQKciBdhw= +github.com/FactomProject/go-bip44 v0.0.0-20190306062959-b541a96d8da9/go.mod h1:4H/Y1yLgSJvRV4iqilsVhYlqXkPgf1QonpKyt6sdP+Q= github.com/Gurpartap/storekit-go v0.0.0-20201205024111-36b6cd5c6a21 h1:HcdvlzaQ4CJfH7xbfJZ3ZHN//BTEpId46iKEMuP3wHE= github.com/Gurpartap/storekit-go v0.0.0-20201205024111-36b6cd5c6a21/go.mod h1:7PODFS++oNZ6khojmPBvkrDeFO/hrc3jmvWvQAOXorw= github.com/KyleBanks/depth v1.2.1 h1:5h8fQADFrWtarTdtDudMmGsC7GPbOAu6RVB3ffsVFHc= @@ -67,7 +98,17 @@ github.com/KyleBanks/depth v1.2.1/go.mod h1:jzSb9d0L43HxTQfT+oSA1EEp2q+ne2uh6Xge github.com/Masterminds/semver/v3 v3.1.1 h1:hLg3sBzpNErnxhQtUy/mmLR2I9foDujNK030IGemrRc= github.com/Masterminds/semver/v3 v3.1.1/go.mod h1:VPu/7SZ7ePZ3QOrcuXROw5FAcLl4a0cBrbBpGY/8hQs= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= +github.com/Shopify/sarama v1.26.1/go.mod h1:NbSGBSSndYaIhRcBtY9V0U7AyH+x71bG668AuWys/yU= +github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMxUHB2q5Ap20/P/eIdh4G0pI= +github.com/StackExchange/wmi v0.0.0-20180116203802-5d049714c4a6/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg= +github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg= +github.com/StackExchange/wmi v0.0.0-20210224194228-fe8f1750fd46/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg= +github.com/StackExchange/wmi v1.2.1/go.mod h1:rcmrprowKIVzvc+NUiLncP2uuArMWLCbu9SBzvHz7e8= +github.com/VictoriaMetrics/fastcache v1.5.7/go.mod h1:ptDBkNMQI4RtmVo8VS/XwRY6RoTu1dAWCbrk+6WsEM8= github.com/VictoriaMetrics/fastcache v1.6.0 h1:C/3Oi3EiBCqufydp1neRZkqcwmEiuRT9c3fqvvgKm5o= +github.com/VictoriaMetrics/fastcache v1.6.0/go.mod h1:0qHz5QP0GMX4pfmMA/zt5RgfNuXJrTP0zS7DqpHGGTw= +github.com/aead/siphash v1.0.1/go.mod h1:Nywa3cDsYNNK3gaciGTWPwHt0wlpNV15vwmswBAUSII= +github.com/ajstarks/svgo v0.0.0-20180226025133-644b8db467af/go.mod h1:K08gAheRH3/J6wwsYMMT4xOr94bZjxIelGM0+d/wbFw= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= @@ -79,7 +120,18 @@ github.com/alexedwards/scs/redisstore v0.0.0-20230217120314-6b1bedc0f08c h1:m2XN github.com/alexedwards/scs/redisstore v0.0.0-20230217120314-6b1bedc0f08c/go.mod h1:ceKFatoD+hfHWWeHOAYue1J+XgOJjE7dw8l3JtIRTGY= github.com/alexedwards/scs/v2 v2.5.0 h1:zgxOfNFmiJyXG7UPIuw1g2b9LWBeRLh3PjfB9BDmfL4= github.com/alexedwards/scs/v2 v2.5.0/go.mod h1:ToaROZxyKukJKT/xLcVQAChi5k6+Pn1Gvmdl7h3RRj8= +github.com/allegro/bigcache v1.2.1-0.20190218064605-e24eb225f156/go.mod h1:Cb/ax3seSYIx7SuZdm2G2xzfwmv3TPSk2ucNfQESPXM= +github.com/allegro/bigcache v1.2.1/go.mod h1:Cb/ax3seSYIx7SuZdm2G2xzfwmv3TPSk2ucNfQESPXM= +github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883/go.mod h1:rCTlJbsFo29Kk6CurOXKm700vrz8f0KW0JNfpkRJY/8= github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= +github.com/antlr/antlr4/runtime/Go/antlr v0.0.0-20210716071054-a231a1a7f1cc h1:0LxTNJTbCStbOMoNcQEc6naaWfU7rI0b+Zdns1EI7TA= +github.com/antlr/antlr4/runtime/Go/antlr v0.0.0-20210716071054-a231a1a7f1cc/go.mod h1:F7bn7fEU90QkQ3tnmaTx3LTKLEDqnwWODIYppRQ5hnY= +github.com/apache/arrow/go/arrow v0.0.0-20191024131854-af6fa24be0db/go.mod h1:VTxUBvSJ3s3eHAg65PNgrsn5BtqCRPdmyXh6rAfdxN0= +github.com/aristanetworks/fsnotify v1.4.2/go.mod h1:D/rtu7LpjYM8tRJphJ0hUBYpjai8SfX+aSNsWDTq/Ks= +github.com/aristanetworks/glog v0.0.0-20191112221043-67e8567f59f3/go.mod h1:KASm+qXFKs/xjSoWn30NrWBBvdTTQq+UjkhjEJHfSFA= +github.com/aristanetworks/goarista v0.0.0-20170210015632-ea17b1a17847/go.mod h1:D/tb0zPVXnP7fmsLZjtdUhSsumbK/ij54UXjjVgMGxQ= +github.com/aristanetworks/goarista v0.0.0-20201012165903-2cb20defcd66/go.mod h1:QZe5Yh80Hp1b6JxQdpfSEEe8X7hTyTEZSosSrFf/oJE= +github.com/aristanetworks/splunk-hec-go v0.3.3/go.mod h1:1VHO9r17b0K7WmOlLb9nTk/2YanvOEnLMUgsFrxBROc= github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= @@ -89,6 +141,16 @@ github.com/attestantio/go-eth2-client v0.15.7 h1:0v7+Z2RZ8bNtU/0mfppXzLiYv+6a8pe github.com/attestantio/go-eth2-client v0.15.7/go.mod h1:/Oh6YTuHmHhgLN/ZnQRKHGc7HdIzGlDkI2vjNZvOsvA= github.com/awa/go-iap v1.3.7 h1:ErmeZRa8I4tx+ToAHikpARoAZVSszHWpwyl4FCj/6XA= github.com/awa/go-iap v1.3.7/go.mod h1:Jq6HjuGiT1FXSp92RDmpnW8c9SzmEqp10fE3FrljmBI= +github.com/aws/aws-sdk-go v1.25.48/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= +github.com/aws/aws-sdk-go-v2 v1.2.0/go.mod h1:zEQs02YRBw1DjK0PoJv3ygDYOFTre1ejlJWl8FwAuQo= +github.com/aws/aws-sdk-go-v2/config v1.1.1/go.mod h1:0XsVy9lBI/BCXm+2Tuvt39YmdHwS5unDQmxZOYe8F5Y= +github.com/aws/aws-sdk-go-v2/credentials v1.1.1/go.mod h1:mM2iIjwl7LULWtS6JCACyInboHirisUUdkBPoTHMOUo= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.0.2/go.mod h1:3hGg3PpiEjHnrkrlasTfxFqUsZ2GCk/fMUn4CbKgSkM= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.0.2/go.mod h1:45MfaXZ0cNbeuT0KQ1XJylq8A6+OpVV2E5kvY/Kq+u8= +github.com/aws/aws-sdk-go-v2/service/route53 v1.1.1/go.mod h1:rLiOUrPLW/Er5kRcQ7NkwbjlijluLsrIbu/iyl35RO4= +github.com/aws/aws-sdk-go-v2/service/sso v1.1.1/go.mod h1:SuZJxklHxLAXgLTc1iFXbEWkXs7QRTQpCLGaKIprQW0= +github.com/aws/aws-sdk-go-v2/service/sts v1.1.1/go.mod h1:Wi0EBZwiz/K44YliU0EKxqTCJGUfYTWXrrBwkq736bM= +github.com/aws/smithy-go v1.1.0/go.mod h1:EzMw8dbp/YJL4A5/sbhGddag+NPT7q084agLbB9LgIw= github.com/aybabtme/uniplot v0.0.0-20151203143629-039c559e5e7e h1:dSeuFcs4WAJJnswS8vXy7YY1+fdlbVPuEVmDAfqvFOQ= github.com/aybabtme/uniplot v0.0.0-20151203143629-039c559e5e7e/go.mod h1:uh71c5Vc3VNIplXOFXsnDy21T1BepgT32c5X/YPrOyc= github.com/bazelbuild/rules_go v0.23.2 h1:Wxu7JjqnF78cKZbsBsARLSXx/jlGaSLCnUV3mTlyHvM= @@ -99,18 +161,39 @@ github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6r github.com/bgentry/go-netrc v0.0.0-20140422174119-9fd32a8b3d3d/go.mod h1:6QX/PXZ00z/TKoufEY6K/a0k6AhaJrQKdFe6OfVXsa4= github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c/go.mod h1:MKsuJmJgSg28kpZDP6UIiPt0e0Oz0kqKNGyRaWEPv84= +github.com/bketelsen/crypt v0.0.4/go.mod h1:aI6NrJ0pMGgvZKL1iVgXLnfIFJtfV+bKCoqOes/6LfM= github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869 h1:DDGfHa7BWjL4YnC6+E63dPcxHo2sUxDIu8g3QgEJdRY= github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869/go.mod h1:Ekp36dRnpXw/yCqJaO+ZrUyxD+3VXMFFr56k5XYrpB4= +github.com/bmizerany/pat v0.0.0-20170815010413-6226ea591a40/go.mod h1:8rLXio+WjiTceGBHIoTvn60HIbs7Hm7bcHjyrSqYB9c= +github.com/boltdb/bolt v1.3.1/go.mod h1:clJnj/oiGkjum5o1McbSZDSLxVThjynRyGBgiAx27Ps= +github.com/btcsuite/btcd v0.0.0-20171128150713-2e60448ffcc6/go.mod h1:Dmm/EzmjnCiweXmzRIAiUWCInVmPgjkzgv5k4tVyXiQ= +github.com/btcsuite/btcd v0.20.1-beta/go.mod h1:wVuoA8VJLEcwgqHBwHmzLRazpKxTv13Px/pDuV7OomQ= +github.com/btcsuite/btcd v0.21.0-beta/go.mod h1:ZSWyehm27aAuS9bvkATT+Xte3hjHZ+MRgMY/8NJ7K94= +github.com/btcsuite/btcd v0.22.0-beta/go.mod h1:9n5ntfhhHQBIhUvlhDvD3Qg6fRUj4jkN0VB8L8svzOA= github.com/btcsuite/btcd v0.23.4 h1:IzV6qqkfwbItOS/sg/aDfPDsjPP8twrCOE2R93hxMlQ= github.com/btcsuite/btcd/btcec/v2 v2.3.2 h1:5n0X6hX0Zk+6omWcihdYvdAlGf2DfasC0GMf7DClJ3U= github.com/btcsuite/btcd/btcec/v2 v2.3.2/go.mod h1:zYzJ8etWJQIv1Ogk7OzpWjowwOdXY1W/17j2MW85J04= github.com/btcsuite/btcd/chaincfg/chainhash v1.0.1 h1:q0rUy8C/TYNBQS1+CGKw68tLOFYSNEs0TFnxxnS9+4U= +github.com/btcsuite/btclog v0.0.0-20170628155309-84c8d2346e9f/go.mod h1:TdznJufoqS23FtqVCzL0ZqgP5MqXbb4fg/WgDys70nA= +github.com/btcsuite/btcutil v0.0.0-20190425235716-9e5f4b9a998d/go.mod h1:+5NJ2+qvTyV9exUAL/rxXi3DcLg2Ts+ymUAY5y4NvMg= +github.com/btcsuite/btcutil v1.0.2/go.mod h1:j9HUFwoQRsZL3V4n+qG+CUnEGHOarIxfC3Le2Yhbcts= +github.com/btcsuite/btcutil v1.0.3-0.20201208143702-a53e38424cce/go.mod h1:0DVlHczLPewLcPGEIeUEzfOJhqGPQ0mJJRDBtD307+o= +github.com/btcsuite/go-socks v0.0.0-20170105172521-4720035b7bfd/go.mod h1:HHNXQzUsZCxOoE+CPiyCTO6x34Zs86zZUiwtpXoGdtg= +github.com/btcsuite/goleveldb v0.0.0-20160330041536-7834afc9e8cd/go.mod h1:F+uVaaLLH7j4eDXPRvw78tMflu7Ie2bzYOH4Y8rRKBY= +github.com/btcsuite/goleveldb v1.0.0/go.mod h1:QiK9vBlgftBg6rWQIj6wFzbPfRjiykIEhBH4obrXJ/I= +github.com/btcsuite/snappy-go v0.0.0-20151229074030-0bdef8d06723/go.mod h1:8woku9dyThutzjeg+3xrA5iCpBRH8XEEg3lh6TiUghc= +github.com/btcsuite/snappy-go v1.0.0/go.mod h1:8woku9dyThutzjeg+3xrA5iCpBRH8XEEg3lh6TiUghc= +github.com/btcsuite/websocket v0.0.0-20150119174127-31079b680792/go.mod h1:ghJtEyQwv5/p4Mg4C0fgbePVuGr935/5ddU9Z3TmDRY= +github.com/btcsuite/winsvc v1.0.0/go.mod h1:jsenWakMcC0zFBFurPLEAyrnc/teJEM1O46fmI40EZs= github.com/bufbuild/buf v0.37.0/go.mod h1:lQ1m2HkIaGOFba6w/aC3KYBHhKEOESP3gaAEpS3dAFM= +github.com/c-bata/go-prompt v0.2.2/go.mod h1:VzqtzE2ksDBcdln8G7mk2RX9QyGjH+OVqOCSiVIqS34= github.com/carlmjohnson/requests v0.23.4 h1:AxcvapfB9RPXLSyvAHk9YJoodQ43ZjzNHj6Ft3tQGdg= github.com/carlmjohnson/requests v0.23.4/go.mod h1:Qzp6tW4DQyainPP+tGwiJTzwxvElTIKm0B191TgTtOA= github.com/census-instrumentation/opencensus-proto v0.2.1 h1:glEXhBS5PSLLv4IXzLA5yPRVX4bilULVyxxbrfOtDAk= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/cespare/cp v0.1.0/go.mod h1:SOGHArjBr4JWaSDEVpWpo/hNg6RoKrls6Oh40hiwW+s= github.com/cespare/cp v1.1.1 h1:nCb6ZLdB7NRaqsm91JtQTAme2SKJzXVsdPIPkyJr1MU= +github.com/cespare/cp v1.1.1/go.mod h1:SOGHArjBr4JWaSDEVpWpo/hNg6RoKrls6Oh40hiwW+s= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= @@ -120,6 +203,9 @@ github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWR github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= +github.com/cloudflare/cloudflare-go v0.10.2-0.20190916151808-a80f83b9add9/go.mod h1:1MxXX1Ux4x6mqPmjkUgTP1CdXIBXKX7T+Jk9Gxrmx+U= +github.com/cloudflare/cloudflare-go v0.14.0/go.mod h1:EnwdgGMaFOruiPZRFSgn+TsQ3hQ7C/YWzIGLeu5c304= +github.com/cmars/basen v0.0.0-20150613233007-fe3947df716e/go.mod h1:P13beTBKr5Q18lJe1rIoLUqjM+CB1zYrRg44ZqGuQSA= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= @@ -135,6 +221,8 @@ github.com/cockroachdb/errors v1.9.1 h1:yFVvsI0VxmRShfawbt/laCIDy/mtTqqnvoNgiy5b github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b h1:r6VH0faHjZeQy818SGhaone5OnYfxFR/+AzdY3sf5aE= github.com/cockroachdb/pebble v0.0.0-20230209160836-829675f94811 h1:ytcWPaNPhNoGMWEhDvS3zToKcDpRsLuRolQJBVGdozk= github.com/cockroachdb/redact v1.1.3 h1:AKZds10rFSIj7qADf0g46UixK8NNLwWTNdCIGS5wfSQ= +github.com/consensys/bavard v0.1.8-0.20210406032232-f3452dc9b572/go.mod h1:Bpd0/3mZuaj6Sj+PqrmIquiOKy397AKGThQPaGzNXAQ= +github.com/consensys/gnark-crypto v0.4.1-0.20210426202927-39ac3d4b3f1f/go.mod h1:815PAHg3wvysy0SyIqanF8gZ0Y1wjk/hrDHD/iT88+Q= github.com/coocood/freecache v1.2.3 h1:lcBwpZrwBZRZyLk/8EMyQVXRiFl663cCuMOrjCALeto= github.com/coocood/freecache v1.2.3/go.mod h1:RBUWa/Cy+OHdfTGFEhEuE1pMCMX51Ncizj7rthiQ3vk= github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= @@ -144,34 +232,57 @@ github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7 github.com/coreos/go-systemd v0.0.0-20190719114852-fd7a80b32e1f/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= +github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/cpuguy83/go-md2man/v2 v2.0.2 h1:p1EgwI/C7NhT0JmVkwCD2ZBK8j4aeHQX2pMHHBfMQ6w= github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/cyberdelia/templates v0.0.0-20141128023046-ca7fffd4298c/go.mod h1:GyV+0YP4qX0UQ7r2MoYZ+AvYDp12OF5yg4q8rGnyNh4= github.com/d4l3k/messagediff v1.2.1 h1:ZcAIMYsUg0EAp9X+tt8/enBE/Q8Yd5kzPynLyKptt9U= +github.com/dave/jennifer v1.2.0/go.mod h1:fIb+770HOpJ2fmN9EPPKOqm1vMGhB+TwXKMZhrIygKg= +github.com/davecgh/go-spew v0.0.0-20171005155431-ecdeabc65495/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/deckarep/golang-set v0.0.0-20180603214616-504e848d77ea/go.mod h1:93vsz/8Wt4joVM7c2AVqh+YRMiUSc14yDtF28KmMOgQ= +github.com/deckarep/golang-set v1.7.1/go.mod h1:93vsz/8Wt4joVM7c2AVqh+YRMiUSc14yDtF28KmMOgQ= github.com/deckarep/golang-set/v2 v2.1.0 h1:g47V4Or+DUdzbs8FxCCmgb6VYd+ptPAngjM6dtGktsI= github.com/deckarep/golang-set/v2 v2.1.0/go.mod h1:VAky9rY/yGXJOLEDv3OMci+7wtDpOF4IN+y82NBOac4= github.com/decred/dcrd/crypto/blake256 v1.0.0 h1:/8DMNYp9SGi5f0w7uCm6d6M4OU2rGFK09Y2A4Xv7EE0= github.com/decred/dcrd/dcrec/secp256k1/v4 v4.1.0 h1:HbphB4TFFXpv7MNrT52FGrrgVXF1owhMVTHFZIlnvd4= github.com/decred/dcrd/dcrec/secp256k1/v4 v4.1.0/go.mod h1:DZGJHZMqrU4JJqFAWUS2UO1+lbSKsdiOoYi9Zzey7Fc= +github.com/decred/dcrd/lru v1.0.0/go.mod h1:mxKOwFd7lFjN2GZYsiz/ecgqR6kkYAl+0pz0tEMk218= +github.com/deepmap/oapi-codegen v1.6.0/go.mod h1:ryDa9AgbELGeB+YEXE1dR53yAjHwFvE9iAUlWl9Al3M= +github.com/deepmap/oapi-codegen v1.8.2/go.mod h1:YLgSKSDv/bZQB7N4ws6luhozi3cEdRktEqrX88CvjIw= github.com/dgraph-io/ristretto v0.1.1 h1:6CWw5tJNgpegArSHpNHJKldNeq03FQCwYvfMVWajOK8= github.com/dgraph-io/ristretto v0.1.1/go.mod h1:S1GPSBCYCIhmVNfcth17y2zZtQT6wzkzgwUve0VDWWA= github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= +github.com/dgryski/go-bitstream v0.0.0-20180413035011-3522498ce2c8/go.mod h1:VMaSuZ+SZcx/wljOQKvp5srsbCiKDEb6K2wC4+PiBmQ= github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2 h1:tdlZCpZ/P9DhczCTSixgIKmwPv6+wP5DGjqLYw5SUiA= github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78= github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc= github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= +github.com/dlclark/regexp2 v1.2.0/go.mod h1:2pZnwuY/m+8K6iRw6wQdMtk+rH5tNGR1i55kozfMjCc= +github.com/dlclark/regexp2 v1.4.1-0.20201116162257-a2a8dda75c91/go.mod h1:2pZnwuY/m+8K6iRw6wQdMtk+rH5tNGR1i55kozfMjCc= +github.com/docker/docker v1.4.2-0.20180625184442-8e610b2b55bf/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= github.com/donovanhide/eventsource v0.0.0-20210830082556-c59027999da0 h1:C7t6eeMaEQVy6e8CarIhscYQlNmw5e3G36y7l7Y21Ao= github.com/donovanhide/eventsource v0.0.0-20210830082556-c59027999da0/go.mod h1:56wL82FO0bfMU5RvfXoIwSOP2ggqqxT+tAfNEIyxuHw= +github.com/dop251/goja v0.0.0-20200721192441-a695b0cdd498/go.mod h1:Mw6PkjjMXWbTj+nnj4s3QPXq1jaT0s5pC0iFD4+BOAA= +github.com/dop251/goja v0.0.0-20211011172007-d99e4b8cbf48/go.mod h1:R9ET47fwRVRPZnOGvHxxhuZcbrMCuiqOz3Rlrh4KSnk= +github.com/dop251/goja_nodejs v0.0.0-20210225215109-d91c329300e7/go.mod h1:hn7BA7c8pLvoGndExHudxTDKZ84Pyvv+90pbBjbTz0Y= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= +github.com/dvyukov/go-fuzz v0.0.0-20200318091601-be3528f3a813/go.mod h1:11Gm+ccJnvAhCNLlf5+cS9KjtbaD5I5zaZpFMsTHWTw= +github.com/eapache/go-resiliency v1.2.0/go.mod h1:kFI+JgMyC7bLPUVY133qvEBtVayf5mFgVsvEsIPBvNs= +github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21/go.mod h1:+020luEh2TKB4/GOp8oxxtq0Daoen/Cii55CzbTV6DU= +github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I= +github.com/eclipse/paho.mqtt.golang v1.2.0/go.mod h1:H9keYFcgq3Qr5OUJm/JZI/i6U7joQ8SYLhZwfeOo6Ts= +github.com/edsrzf/mmap-go v0.0.0-20160512033002-935e0e8a636c/go.mod h1:YO35OhQPt3KJa3ryjFM5Bs14WD66h8eGKpfaBNrHW5M= github.com/edsrzf/mmap-go v1.0.0 h1:CEBF7HpRnUCSJgGUb5h1Gm7e3VkmVDrR8lvWVLtrOFw= +github.com/edsrzf/mmap-go v1.0.0/go.mod h1:YO35OhQPt3KJa3ryjFM5Bs14WD66h8eGKpfaBNrHW5M= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= @@ -182,6 +293,9 @@ github.com/envoyproxy/go-control-plane v0.10.2-0.20220325020618-49ff273808a1 h1: github.com/envoyproxy/go-control-plane v0.10.2-0.20220325020618-49ff273808a1/go.mod h1:KJwIaB5Mv44NWtYuAOFCVOjcI94vtpEz2JU/D2v6IjE= github.com/envoyproxy/protoc-gen-validate v0.1.0 h1:EQciDnbrYxy13PgWoY8AqoxGiPrpgBZ1R8UNe3ddc+A= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= +github.com/ethereum/go-ethereum v1.9.23/go.mod h1:JIfVb6esrqALTExdz9hRYvrP0xBDf6wCncIu1hNwHpM= +github.com/ethereum/go-ethereum v1.10.4/go.mod h1:nEE0TP5MtxGzOMd7egIrbPJMQBnhVU3ELNxhBglIzhg= +github.com/ethereum/go-ethereum v1.10.12/go.mod h1:W3yfrFyL9C1pHcwY5hmRHVDaorTiQxhYBkKyu5mEDHw= github.com/ethereum/go-ethereum v1.11.3 h1:uuBkYUJW9aY5JYi3+sqLHz+XWyo5fmn/ab9XcbtVDTU= github.com/ethereum/go-ethereum v1.11.3/go.mod h1:rBUvAl5cdVrAei9q5lgOU7RSEuPJk1nlBDnS/YSoKQE= github.com/evanw/esbuild v0.8.23 h1:eRRG1fNtQ9KPG3lM62EUYagLVMSuxSTBEgukqY0et3w= @@ -192,28 +306,44 @@ github.com/facebookgo/stack v0.0.0-20160209184415-751773369052 h1:JWuenKqqX8nojt github.com/facebookgo/stack v0.0.0-20160209184415-751773369052/go.mod h1:UbMTZqLaRiH3MsBH8va0n7s1pQYcu3uTb8G4tygF4Zg= github.com/facebookgo/subset v0.0.0-20150612182917-8dac2c3c4870 h1:E2s37DuLxFhQDg5gKsWoLBOB0n+ZW8s599zru8FJ2/Y= github.com/facebookgo/subset v0.0.0-20150612182917-8dac2c3c4870/go.mod h1:5tD+neXqOorC30/tWg0LCSkrqj/AR6gu8yY8/fpw1q0= +github.com/fatih/color v1.3.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/fatih/color v1.10.0/go.mod h1:ELkj/draVOlAH/xkhN6mQ50Qd0MPOk5AAr3maGEBuJM= github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk= github.com/fatih/color v1.14.1 h1:qfhVLaG5s+nCROl1zJsZRxFeYrHLqWroPOQ8BWiNb4w= github.com/fatih/color v1.14.1/go.mod h1:2oHN61fhTpgcxD3TSWCgKDiH1+x4OiDVVGH8WlgGZGg= +github.com/ferranbt/fastssz v0.0.0-20210120143747-11b9eff30ea9/go.mod h1:DyEu2iuLBnb/T51BlsiO3yLYdJC6UbGMrIkqK1KmQxM= +github.com/ferranbt/fastssz v0.0.0-20210905181407-59cf6761a7d5/go.mod h1:S8yiDeAXy8f88W4Ul+0dBMPx49S05byYbmZD6Uv94K4= github.com/ferranbt/fastssz v0.1.2/go.mod h1:X5UPrE2u1UJjxHA8X54u04SBwdAQjG2sFtWs39YxyWs= github.com/ferranbt/fastssz v0.1.3 h1:ZI+z3JH05h4kgmFXdHuR1aWYsgrg7o+Fw7/NCzM16Mo= github.com/ferranbt/fastssz v0.1.3/go.mod h1:0Y9TEd/9XuFlh7mskMPfXiI2Dkw4Ddg9EyXt1W7MRvE= +github.com/fjl/memsize v0.0.0-20180418122429-ca190fb6ffbc/go.mod h1:VvhXpOYNQvB+uIk2RvXzuaQtkQJzzIx6lSBe1xv7hi0= github.com/fjl/memsize v0.0.0-20190710130421-bcb5799ab5e5 h1:FtmdgXiUlNeRsoNMFlKLDt+S+6hbjVMEW6RGQ7aUf7c= +github.com/fjl/memsize v0.0.0-20190710130421-bcb5799ab5e5/go.mod h1:VvhXpOYNQvB+uIk2RvXzuaQtkQJzzIx6lSBe1xv7hi0= +github.com/fogleman/gg v1.2.1-0.20190220221249-0403632d5b90/go.mod h1:R/bRT+9gY/C5z7JzPU0zXsXHKM4/ayA+zqcVNZzPa1k= +github.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g= +github.com/frankban/quicktest v1.7.2/go.mod h1:jaStnuzAqU1AJdCO0l53JDCJrVDKcS03DbaAcR7Ks/o= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= github.com/fsnotify/fsnotify v1.5.4/go.mod h1:OVB6XrOHzAwXMpEM7uPOzcehqUV2UqJxmVXmkdnm1bU= github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY= github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw= +github.com/garyburd/redigo v1.6.0/go.mod h1:NR3MbYisc3/PwhQ00EMzDiPmrwpPxAn5GI05/YaO1SY= +github.com/gballet/go-libpcsclite v0.0.0-20190607065134-2772fd86a8ff/go.mod h1:x7DCsMOv1taUwEWCzT4cmDeAkigA5/QCwUodaVOe8Ww= github.com/gballet/go-libpcsclite v0.0.0-20191108122812-4678299bea08 h1:f6D9Hr8xV8uYKlyuj8XIruxlh9WjVjdh1gIicAS7ays= +github.com/gballet/go-libpcsclite v0.0.0-20191108122812-4678299bea08/go.mod h1:x7DCsMOv1taUwEWCzT4cmDeAkigA5/QCwUodaVOe8Ww= +github.com/getkin/kin-openapi v0.53.0/go.mod h1:7Yn5whZr5kJi6t+kShccXS8ae1APpYTW6yheSwk8Yi4= +github.com/getkin/kin-openapi v0.61.0/go.mod h1:7Yn5whZr5kJi6t+kShccXS8ae1APpYTW6yheSwk8Yi4= github.com/getsentry/sentry-go v0.18.0 h1:MtBW5H9QgdcJabtZcuJG80BMOwaBpkRDZkxRkNC1sN0= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm+fLHvGI= github.com/gin-gonic/gin v1.5.0/go.mod h1:Nd6IXA8m5kNZdNEHMBd93KT+mdY3+bewLgRvmCsR2Do= +github.com/glycerine/go-unsnap-stream v0.0.0-20180323001048-9f0cb55181dd/go.mod h1:/20jfyN9Y5QPEAprSgKAUr+glWDY39ZiUEAYOEv5dsE= +github.com/glycerine/goconvey v0.0.0-20190410193231-58a59202ab31/go.mod h1:Ogl1Tioa0aV7gstGFO7KhffUsb9M4ydbEbbxpcEDc24= github.com/go-chi/chi v4.0.0+incompatible/go.mod h1:eB3wogJHnLi3x/kFX2A+IbTBlXxmMeXJVKy9tTv1XzQ= github.com/go-chi/chi v4.0.2+incompatible h1:maB6vn6FqCxrpz4FqWdh4+lwpyZIQS7YEAUcHlgXVRs= github.com/go-chi/chi v4.0.2+incompatible/go.mod h1:eB3wogJHnLi3x/kFX2A+IbTBlXxmMeXJVKy9tTv1XzQ= +github.com/go-chi/chi/v5 v5.0.0/go.mod h1:BBug9lr0cqtdAhsu6R4AAdvufI0/XBzAQSsUqJpoZOs= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= @@ -224,6 +354,9 @@ github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9 github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= github.com/go-logr/logr v1.2.3 h1:2DntVwHkVopvECVRSlL5PSo9eG+cAkDCuckLubN+rq0= +github.com/go-ole/go-ole v1.2.1/go.mod h1:7FAglXiTm7HKlQRDeOQ6ZNUHidzCWXuZWq/1dTyBNF8= +github.com/go-ole/go-ole v1.2.4/go.mod h1:XCwSNxSkXRo4vlyPy93sltvi/qJq0jqQhjqQNIwKuxM= +github.com/go-ole/go-ole v1.2.5/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY= github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= @@ -248,7 +381,10 @@ github.com/go-playground/validator/v10 v10.4.1/go.mod h1:nlOn6nFhuKACm19sB/8EGNn github.com/go-playground/validator/v10 v10.10.0 h1:I7mrTYv78z8k8VXa/qJlOlEXn/nBh+BF8dHX5nt/dr0= github.com/go-redis/redis/v8 v8.11.5 h1:AcZZR7igkdvfVmQTPnu9WE37LRrO/YrBH5zWyjDC0oI= github.com/go-redis/redis/v8 v8.11.5/go.mod h1:gREzHqY1hg6oD9ngVRbLStwAWKhA0FEgq8Jd4h5lpwo= +github.com/go-sourcemap/sourcemap v2.1.2+incompatible/go.mod h1:F8jJfvm2KbVjc5NqelyYJmf/v5J0dwNLS2mL4sNA1Jg= +github.com/go-sourcemap/sourcemap v2.1.3+incompatible/go.mod h1:F8jJfvm2KbVjc5NqelyYJmf/v5J0dwNLS2mL4sNA1Jg= github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= +github.com/go-sql-driver/mysql v1.4.1/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= github.com/go-sql-driver/mysql v1.7.0 h1:ueSltNNllEqE3qcWBTD0iQd3IpL/6U+mJxLkazJ7YPc= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/go-stack/stack v1.8.1 h1:ntEHSVwIt7PNXNpgPmVfMrNhLtgjlmnZha2kOpuRiDw= @@ -266,14 +402,19 @@ github.com/goccy/go-yaml v1.10.0/go.mod h1:h/18Lr6oSQ3mvmqFoWmQ47KChOgpfHpTyIHl3 github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/gofrs/flock v0.8.0/go.mod h1:F1TvTiK9OcQqauNUHlbJvyl9Qa1QvF/gOUDKA14jxHU= github.com/gofrs/flock v0.8.1 h1:+gYjHKf32LDeiEEFhQaotPbLuUXjY5ZqxKgXy7n59aw= +github.com/gofrs/uuid v3.3.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= github.com/gofrs/uuid v4.0.0+incompatible h1:1SD/1F5pU8p29ybwgQSwpQk+mwdRrXCYuPhW6m+TnJw= github.com/gofrs/uuid v4.0.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= +github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= +github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang-jwt/jwt v3.2.2+incompatible h1:IfV12K8xAKAnZqdXVzCZ+TOjboZ2keLg81eXfW3O+oY= github.com/golang-jwt/jwt v3.2.2+incompatible/go.mod h1:8pz2t5EyA70fFQQSrl6XZXzqecmYZeUEB8OUGHkxJ+I= github.com/golang-jwt/jwt/v4 v4.3.0 h1:kHL1vqdqWNfATmA0FNMdmZNMyZI1U6O31X4rlIPoBog= +github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0/go.mod h1:E/TSTwGwJL78qG/PmXZO1EjYhfJinVAhrmmHX6Z8B9k= +github.com/golang/geo v0.0.0-20190916061304-5b978397cfec/go.mod h1:QZ0nwyI2jOfgRAoBvP+ab5aRr7c9x7lhGEJrKvBwjWI= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/glog v1.0.0 h1:nfP3RFugxnNRyKgeWd4oI1nYvXpxrx8ck8ZrcizshdQ= github.com/golang/glog v1.0.0/go.mod h1:EWib/APOK0SL3dFbYqvxE3UYd8E6s1ouQ7iEp/0LWV4= @@ -290,6 +431,7 @@ github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4= +github.com/golang/mock v1.5.0/go.mod h1:CWnOUgYIOo4TcNZ0wHX3YZCqsaM1I1Jvs6v3mP3KVu8= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= @@ -305,16 +447,23 @@ github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QD github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/golang/protobuf v1.5.1/go.mod h1:DopwsBzvsk0Fs44TXzsVbJyPhcCPeIwnvohx4u74HPM= github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/golang/snappy v0.0.2-0.20200707131729-196ae77b8a26/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/golang/snappy v0.0.2/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/golangci/lint-1 v0.0.0-20181222135242-d2cdd8c08219/go.mod h1:/X8TswGSh1pIozq4ZwCfxS0WA5JGXguxk94ar/4c87Y= github.com/gomodule/redigo v1.8.0 h1:OXfLQ/k8XpYF8f8sZKd2Df4SDyzbLeC35OsBsB11rYg= github.com/gomodule/redigo v1.8.0/go.mod h1:P9dn9mFrCBvWhGE1wpxx6fgq7BAeLBk+UUUzlpkBYO0= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.1.2 h1:xf4v41cLI2Z6FxbKm+8Bu+m8ifhj15JuZ9sa0jZCMUU= +github.com/google/flatbuffers v1.11.0/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= @@ -330,6 +479,7 @@ github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/gofuzz v1.1.1-0.20200604201612-c04b05f3adfa/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= github.com/google/martian v2.1.0+incompatible h1:/CP5g8u/VJHijgedC/Legn3BAbAaWPgecwXBIDzw5no= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= @@ -345,9 +495,14 @@ github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hf github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20201023163331-3e6fc7fc9c4c/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20201203190320-1bf35d6f28c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20210122040257-d980be63207e/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20210226084205-cbba55b83ad5/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= +github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.1.5/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.2.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/enterprise-certificate-proxy v0.2.0 h1:y8Yozv7SZtlU//QXbezB6QkpuE6jMD2/gfzk4AftXjs= @@ -366,9 +521,12 @@ github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI= github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= github.com/gorilla/securecookie v1.1.1 h1:miw7JPhV+b/lAHSXz4qd/nN9jRiAFV5FwjeKyCS8BvQ= github.com/gorilla/securecookie v1.1.1/go.mod h1:ra0sb63/xPlUeL+yeDciTfxMRAA+MP+HVt/4epWDjd4= +github.com/gorilla/websocket v1.4.1-0.20190629185528-ae1634f6a989/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/gorilla/websocket v1.5.0 h1:PPwGk2jz7EePpoHN/+ClbZu8SPxiqlu12wZP/3sWmnc= github.com/gorilla/websocket v1.5.0/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= +github.com/graph-gophers/graphql-go v0.0.0-20191115155744-f33e81362277/go.mod h1:9CQHMSxwO4MprSdzoIEobiHpoLtHm77vfxsvsIN5Vuc= +github.com/graph-gophers/graphql-go v0.0.0-20201113091052-beb923fada29/go.mod h1:9CQHMSxwO4MprSdzoIEobiHpoLtHm77vfxsvsIN5Vuc= github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= @@ -377,6 +535,7 @@ github.com/hashicorp/consul/api v1.1.0/go.mod h1:VmuI/Lkw1nC05EYQWNKwWGbkg+FbDBt github.com/hashicorp/consul/sdk v0.1.1/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8= github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/go-bexpr v0.1.10 h1:9kuI5PFotCboP3dkDYFr/wi0gg0QVbSNz5oFRpxn4uE= +github.com/hashicorp/go-bexpr v0.1.10/go.mod h1:oxlubA2vC/gFVfX1A6JGp7ls7uCDlfJn732ehYYg+g0= github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM= @@ -386,31 +545,59 @@ github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerX github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4= github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= +github.com/hashicorp/go-uuid v1.0.2/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= github.com/hashicorp/go-version v1.6.0 h1:feTTfFNnjP967rlCxM/I9g701jU+RN74YKx2mOkIeek= github.com/hashicorp/go-version v1.6.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= github.com/hashicorp/golang-lru v0.5.5-0.20210104140557-80c98217689d h1:dg1dEPuWpEqDnvIw251EVy4zlP8gWbsGj4BsUKCRpYs= github.com/hashicorp/golang-lru v0.5.5-0.20210104140557-80c98217689d/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= +github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64= github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0mNTz8vQ= github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I= github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc= github.com/herumi/bls-eth-go-binary v0.0.0-20200522010937-01d282b5380b/go.mod h1:luAnRm3OsMQeokhGzpYmc0ZKwawY7o87PUEP11Z7r7U= +github.com/herumi/bls-eth-go-binary v0.0.0-20210208203315-e81c3e745d31/go.mod h1:luAnRm3OsMQeokhGzpYmc0ZKwawY7o87PUEP11Z7r7U= +github.com/herumi/bls-eth-go-binary v0.0.0-20210917013441-d37c07cfda4e/go.mod h1:luAnRm3OsMQeokhGzpYmc0ZKwawY7o87PUEP11Z7r7U= +github.com/herumi/bls-eth-go-binary v1.28.1/go.mod h1:luAnRm3OsMQeokhGzpYmc0ZKwawY7o87PUEP11Z7r7U= github.com/herumi/bls-eth-go-binary v1.29.1 h1:XcNSHYTyNjEUVfWDCE2gtG5r95biTwd7MJUJF09LtSE= github.com/herumi/bls-eth-go-binary v1.29.1/go.mod h1:luAnRm3OsMQeokhGzpYmc0ZKwawY7o87PUEP11Z7r7U= github.com/holiman/big v0.0.0-20221017200358-a027dc42d04e h1:pIYdhNkDh+YENVNi3gto8n9hAmRxKxoar0iE6BLucjw= github.com/holiman/bloomfilter/v2 v2.0.3 h1:73e0e/V0tCydx14a0SCYS/EWCxgwLZ18CZcZKVu0fao= +github.com/holiman/bloomfilter/v2 v2.0.3/go.mod h1:zpoh+gs7qcpqrHr3dB55AMiJwo0iURXE7ZOP9L9hSkA= +github.com/holiman/uint256 v1.1.1/go.mod h1:y4ga/t+u+Xwd7CpDgZESaRcWy0I7XMlTMA25ApIH5Jw= github.com/holiman/uint256 v1.2.0 h1:gpSYcPLWGv4sG43I2mVLiDZCNDh/EpGjSk8tmtxitHM= +github.com/holiman/uint256 v1.2.0/go.mod h1:y4ga/t+u+Xwd7CpDgZESaRcWy0I7XMlTMA25ApIH5Jw= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= +github.com/huin/goupnp v1.0.0/go.mod h1:n9v9KO1tAxYH82qOn+UTIFQDmx5n1Zxd/ClZDMX7Bnc= +github.com/huin/goupnp v1.0.1-0.20210310174557-0ca763054c88/go.mod h1:nNs7wvRfN1eKaMknBydLNQU6146XQim8t4h+q90biWo= +github.com/huin/goupnp v1.0.2/go.mod h1:0dxJBVBHqTMjIUMkESDTNgOOx/Mw5wYIfyFmdzSamkM= github.com/huin/goupnp v1.0.3 h1:N8No57ls+MnjlB+JPiCVSOyy/ot7MJTqlo7rn+NYSqQ= +github.com/huin/goutil v0.0.0-20170803182201-1ca381bf3150/go.mod h1:PpLOETDnJ0o3iZrZfqZzyLl6l7F3c6L1oWn7OICBi6o= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/imdario/mergo v0.3.13 h1:lFzP57bqS/wsqKssCGmtLAb8A0wKjLGrve2q3PPVcBk= github.com/imdario/mergo v0.3.13/go.mod h1:4lJ1jqUDcsbIECGy0RUJAXNIhg+6ocWgb1ALK2O4oXg= +github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= +github.com/influxdata/flux v0.65.1/go.mod h1:J754/zds0vvpfwuq7Gc2wRdVwEodfpCFM7mYlOw2LqY= +github.com/influxdata/influxdb v1.2.3-0.20180221223340-01288bdb0883/go.mod h1:qZna6X/4elxqT3yI9iZYdZrWWdeFOOprn86kgg4+IzY= +github.com/influxdata/influxdb v1.8.3/go.mod h1:JugdFhsvvI8gadxOI6noqNeeBHvWNTbfYGtiAn+2jhI= +github.com/influxdata/influxdb-client-go/v2 v2.4.0/go.mod h1:vLNHdxTJkIf2mSLvGrpj8TCcISApPoXkaxP8g9uRlW8= +github.com/influxdata/influxdb1-client v0.0.0-20191209144304-8bf82d3c094d/go.mod h1:qj24IKcXYK6Iy9ceXlo3Tc+vtHo9lIhSX5JddghvEPo= +github.com/influxdata/influxql v1.1.1-0.20200828144457-65d3ef77d385/go.mod h1:gHp9y86a/pxhjJ+zMjNXiQAA197Xk9wLxaz+fGG+kWk= +github.com/influxdata/line-protocol v0.0.0-20180522152040-32c6aa80de5e/go.mod h1:4kt73NQhadE3daL3WhR5EJ/J2ocX0PZzwxQ0gXJ7oFE= +github.com/influxdata/line-protocol v0.0.0-20200327222509-2487e7298839/go.mod h1:xaLFMmpvUxqXtVkUJfg9QmT88cDaCJ3ZKgdZ78oO8Qo= +github.com/influxdata/line-protocol v0.0.0-20210311194329-9aa0e372d097/go.mod h1:xaLFMmpvUxqXtVkUJfg9QmT88cDaCJ3ZKgdZ78oO8Qo= +github.com/influxdata/promql/v2 v2.12.0/go.mod h1:fxOPu+DY0bqCTCECchSRtWfc+0X19ybifQhZoQNF5D8= +github.com/influxdata/roaring v0.4.13-0.20180809181101-fc520f41fab6/go.mod h1:bSgUQ7q5ZLSO+bKBGqJiCBGAl+9DxyW63zLTujjUlOE= +github.com/influxdata/tdigest v0.0.0-20181121200506-bf2b5ad3c0a9/go.mod h1:Js0mqiSBE6Ffsg94weZZ2c+v/ciT8QRHFOap7EKDrR0= +github.com/influxdata/usage-client v0.0.0-20160829180054-6d3895376368/go.mod h1:Wbbw6tYNvwa5dlB6304Sd+82Z3f7PmVZHVKU637d4po= +github.com/ipfs/go-cid v0.0.7/go.mod h1:6Ux9z5e+HpkQdckYoX1PG/6xqKspzlEIR5SDmgqgC/I= github.com/ipfs/go-cid v0.3.2 h1:OGgOd+JCFM+y1DjWPmVH+2/4POtpDzwcr7VgnB7mZXc= github.com/ipfs/go-cid v0.3.2/go.mod h1:gQ8pKqT/sUxGY+tIwy1RPpAojYu7jAyCp5Tz1svoupw= github.com/jackc/chunkreader v1.0.0/go.mod h1:RT6O25fNZIuasFJRyZ4R/Y2BbhasbmZXF9QQ7T3kePo= @@ -462,45 +649,80 @@ github.com/jackc/puddle v0.0.0-20190608224051-11cab39313c9/go.mod h1:m4B5Dj62Y0f github.com/jackc/puddle v1.1.3/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= github.com/jackc/puddle v1.3.0 h1:eHK/5clGOatcjX3oWGBO/MpxpbHzSwud5EWTSCI+MX0= github.com/jackc/puddle v1.3.0/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= +github.com/jackpal/go-nat-pmp v1.0.2-0.20160603034137-1fa385a6f458/go.mod h1:QPH045xvCAeXUZOxsnwmrtiCoxIr9eob+4orBN1SBKc= github.com/jackpal/go-nat-pmp v1.0.2 h1:KzKSgb7qkJvOUTqYl9/Hg/me3pWgBmERKrTGD7BdWus= +github.com/jackpal/go-nat-pmp v1.0.2/go.mod h1:QPH045xvCAeXUZOxsnwmrtiCoxIr9eob+4orBN1SBKc= +github.com/jcmturner/gofork v1.0.0/go.mod h1:MK8+TM0La+2rjBD4jE12Kj1pCCxK7d2LK/UM3ncEo0o= +github.com/jedisct1/go-minisign v0.0.0-20190909160543-45766022959e/go.mod h1:G1CVv03EnqU1wYL2dFwXxW2An0az9JTl/ZsqXQeBlkU= +github.com/jessevdk/go-flags v0.0.0-20141203071132-1679536dcc89/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= +github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= github.com/jhump/protoreflect v1.8.1/go.mod h1:7GcYQDdMU/O/BBrl/cX6PNHpXh6cenjd8pneu5yW7Tg= +github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= +github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= +github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= github.com/jmoiron/sqlx v1.2.0 h1:41Ip0zITnmWNR/vHV+S4m+VoUivnWY5E4OJfLZjCJMA= github.com/jmoiron/sqlx v1.2.0/go.mod h1:1FEQNm3xlJgrMD+FBdI9+xvCksHtbpVBBw5dYhBSsks= github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= +github.com/jrick/logrotate v1.0.0/go.mod h1:LNinyqDIJnpAur+b8yyulnQw/wDuN1+BYKlTRt3OuAQ= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= +github.com/jsternberg/zap-logfmt v1.0.0/go.mod h1:uvPs/4X51zdkcm5jXl5SYoN+4RK21K8mysFmDaM/h+o= github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/juliangruber/go-intersect v1.1.0 h1:sc+y5dCjMMx0pAdYk/N6KBm00tD/f3tq+Iox7dYDUrY= github.com/juliangruber/go-intersect v1.1.0/go.mod h1:WMau+1kAmnlQnKiikekNJbtGtfmILU/mMU6H7AgKbWQ= +github.com/julienschmidt/httprouter v1.1.1-0.20170430222011-975b5c4c7c21/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= github.com/jung-kurt/gofpdf v1.0.3-0.20190309125859-24315acbbda5 h1:PJr+ZMXIecYc1Ey2zucXdR73SMBtgjPgwa31099IMv0= github.com/jung-kurt/gofpdf v1.0.3-0.20190309125859-24315acbbda5/go.mod h1:7Id9E/uU8ce6rXgefFLlgrJj/GYY22cpxn+r32jIOes= +github.com/jwilder/encoding v0.0.0-20170811194829-b4e1701a28ef/go.mod h1:Ct9fl0F6iIOGgxJ5npU/IUOhOhqlVrGjyIZc8/MagT0= +github.com/karalabe/usb v0.0.0-20190919080040-51dc0efba356/go.mod h1:Od972xHfMJowv7NGVDiWVxk2zxnWgjLlJzE+F4F7AGU= +github.com/karalabe/usb v0.0.0-20191104083709-911d15fe12a9/go.mod h1:Od972xHfMJowv7NGVDiWVxk2zxnWgjLlJzE+F4F7AGU= +github.com/karalabe/usb v0.0.0-20210518091819-4ea20957c210/go.mod h1:Od972xHfMJowv7NGVDiWVxk2zxnWgjLlJzE+F4F7AGU= +github.com/karalabe/usb v0.0.0-20211005121534-4c5740d64559/go.mod h1:Od972xHfMJowv7NGVDiWVxk2zxnWgjLlJzE+F4F7AGU= +github.com/karalabe/usb v0.0.2 h1:M6QQBNxF+CQ8OFvxrT90BA0qBOXymndZnk5q235mFc4= +github.com/karalabe/usb v0.0.2/go.mod h1:Od972xHfMJowv7NGVDiWVxk2zxnWgjLlJzE+F4F7AGU= github.com/kataras/i18n v0.0.5 h1:X9EQHxDhjpN0zh+Ry0PZvi0ODi9lf5mo4wiXWtOYhlY= github.com/kataras/i18n v0.0.5/go.mod h1:U0aKF7ANqGmFVs4WCexDTYGf8wg7Rb3mLJCmr/OuDoo= github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 h1:Z9n2FFNUXsshfwJMBgNA0RU6/i7WVaAegv3PtuIHPMs= github.com/kelseyhightower/envconfig v1.4.0 h1:Im6hONhd3pLkfDFsbRgu68RDNkGF1r3dvMUtDTo2cv8= github.com/kelseyhightower/envconfig v1.4.0/go.mod h1:cccZRl6mQpaq41TPp5QxidR+Sa3axMbJDNb//FQX6Gg= +github.com/kilic/bls12-381 v0.1.0 h1:encrdjqKMEvabVQ7qYOKu1OvhqpK4s47wDYtNiPtlp4= +github.com/kilic/bls12-381 v0.1.0/go.mod h1:vDTTHJONJ6G+P2R74EhnyotQDTliQDnFEwhdmfzw1ig= github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= +github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= +github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/kkdai/bstream v0.0.0-20161212061736-f391b8402d23/go.mod h1:J+Gs4SYgM6CZQHDETBtE9HaSEkGmuNXF86RwHhHUvq4= +github.com/klauspost/compress v1.4.0/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= +github.com/klauspost/compress v1.9.8/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= +github.com/klauspost/compress v1.10.1/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= github.com/klauspost/compress v1.11.7/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= github.com/klauspost/compress v1.16.0 h1:iULayQNOReoYUe+1qtKOqw9CwJv3aNQu8ivo7lw1HU4= github.com/klauspost/compress v1.16.0/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= +github.com/klauspost/cpuid v0.0.0-20170728055534-ae7887de9fa5/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= +github.com/klauspost/cpuid v1.2.3/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= github.com/klauspost/cpuid/v2 v2.0.4/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= +github.com/klauspost/cpuid/v2 v2.0.6/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/klauspost/cpuid/v2 v2.1.2/go.mod h1:RVVoqg1df56z8g3pUjL/3lE5UfnlrJX8tyFgg4nqhuY= github.com/klauspost/cpuid/v2 v2.2.4 h1:acbojRNwl3o09bUq+yDCtZFc1aiwaAAxtcn8YkZXnvk= github.com/klauspost/cpuid/v2 v2.2.4/go.mod h1:RVVoqg1df56z8g3pUjL/3lE5UfnlrJX8tyFgg4nqhuY= +github.com/klauspost/crc32 v0.0.0-20161016154125-cb6bfca970f6/go.mod h1:+ZoRqAPRLkC4NPOvfYeR5KNOrY6TD+/sAC3HXPZgDYg= +github.com/klauspost/pgzip v1.0.2-0.20170402124221-0bf5dcad4ada/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs= github.com/klauspost/pgzip v1.2.5/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs= +github.com/klauspost/reedsolomon v1.9.3/go.mod h1:CwCi+NUr9pqSVktrkN+Ondf06rkhYZ/pcNv7fu+8Un4= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/pty v1.1.8/go.mod h1:O1sed60cT9XZ5uDucP5qwvh+TE3NnUj51EiZO/lmSfw= @@ -508,6 +730,10 @@ github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= +github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= +github.com/labstack/echo/v4 v4.2.1/go.mod h1:AA49e0DZ8kk5jTOOCKNuPR6oTnBS0dYiM4FW1e6jwpg= +github.com/labstack/gommon v0.3.0/go.mod h1:MULnywXg0yavhxWKc+lOruYdAhDwPK9wf0OL7NoOu+k= +github.com/leanovate/gopter v0.2.9/go.mod h1:U2L/78B+KVFIx2VmW6onHJQzXtFb+p5y3y2Sh+Jxxv8= github.com/leodido/go-urn v1.1.0/go.mod h1:+cyI34gQWZcE1eQU7NVgKkkzdXDQHr1dBMtdAPozLkw= github.com/leodido/go-urn v1.2.0/go.mod h1:+8+nEpDfqqsY+g338gtMEUOtuK+4dEMhiQEgxpxOKII= github.com/leodido/go-urn v1.2.1 h1:BqpAaACuzVSgi/VLzGZIobT2z4v53pjosyNd9Yv6n/w= @@ -517,7 +743,10 @@ github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/lib/pq v1.10.2/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= github.com/lib/pq v1.10.7 h1:p7ZhMD+KsSRozJr34udlUrhboJwWAgCg34+/ZZNvZZw= github.com/lib/pq v1.10.7/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= +github.com/libp2p/go-maddr-filter v0.1.0/go.mod h1:VzZhTXkMucEGGEOSKddrwGiOv0tUhgnKqNEmIAz/bPU= github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= +github.com/magiconair/properties v1.8.5 h1:b6kJs+EmPFMYGkow9GiUyCyOvIwYetYJ3fSaWak/Gls= +github.com/magiconair/properties v1.8.5/go.mod h1:y3VJvCyxH9uVvJTWEGAELF3aiYNyPKd5NZ3oSwXrF60= github.com/mailgun/mailgun-go/v4 v4.1.3 h1:KLa5EZaOMMeyvY/lfAhWxv9ealB3mtUsMz0O9XmTtP0= github.com/mailgun/mailgun-go/v4 v4.1.3/go.mod h1:R9kHUQBptF4iSEjhriCQizplCDwrnDShy8w/iPiOfaM= github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= @@ -526,32 +755,54 @@ github.com/mailru/easyjson v0.7.0/go.mod h1:KAzv3t3aY1NaHWoQz1+4F1ccyAH66Jk7yos7 github.com/mailru/easyjson v0.7.6/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= +github.com/matryer/moq v0.0.0-20190312154309-6cfb0558e1bd/go.mod h1:9ELz6aaclSIGnZBoaSLZ3NAl1VTufbOrXBPvtcy6WiQ= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= +github.com/mattn/go-colorable v0.1.0/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcnceauSikq3lYCQ= +github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= github.com/mattn/go-colorable v0.1.6/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= +github.com/mattn/go-colorable v0.1.7/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= github.com/mattn/go-colorable v0.1.8/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= github.com/mattn/go-colorable v0.1.9/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= +github.com/mattn/go-ieproxy v0.0.0-20190610004146-91bb50d98149/go.mod h1:31jz6HNzdxOmlERGGEc4v/dMssOfmp2p5bT/okiKFFc= +github.com/mattn/go-ieproxy v0.0.0-20190702010315-6dee0af9227d/go.mod h1:31jz6HNzdxOmlERGGEc4v/dMssOfmp2p5bT/okiKFFc= github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= +github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= +github.com/mattn/go-isatty v0.0.5-0.20180830101745-3fb116b82035/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.7/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= +github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.9/go.mod h1:YNRxwqDuOph6SZLI9vUUz6OYw3QyUt7WiY2yME+cCiQ= github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= github.com/mattn/go-isatty v0.0.17 h1:BTarxUcIeDqL27Mc+vyvdWYSL28zpIhv3RoTdsLMPng= github.com/mattn/go-isatty v0.0.17/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= +github.com/mattn/go-runewidth v0.0.3/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= +github.com/mattn/go-runewidth v0.0.4/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= +github.com/mattn/go-runewidth v0.0.7/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= +github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= +github.com/mattn/go-runewidth v0.0.13/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= github.com/mattn/go-runewidth v0.0.14 h1:+xnbZSEeDbOIg5/mE6JF0w6n9duR1l3/WmbinWVwUuU= github.com/mattn/go-sqlite3 v1.9.0 h1:pDRiWfl+++eC2FEFRy6jXmQlvp4Yh3z1MJKg4UeYM/4= github.com/mattn/go-sqlite3 v1.9.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= +github.com/mattn/go-sqlite3 v1.11.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= +github.com/mattn/go-tty v0.0.0-20180907095812-13ff1204f104/go.mod h1:XPvLUNfbS4fJH25nqRHfWLMa1ONC8Amw+mIA639KxkE= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo= github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= +github.com/miekg/dns v1.1.43/go.mod h1:+evo5L0630/F6ca/Z9+GAqzhjGyn8/c+TBaOyfEl0V4= +github.com/miekg/dns v1.1.50 h1:DQUfb9uc6smULcREF09Uc+/Gd46YWqJd5DbpPE9xkcA= +github.com/miekg/dns v1.1.50/go.mod h1:e3IlAVfNqAllflbibAZEWOXOQ+Ynzk/dDozDxY7XnME= +github.com/minio/blake2b-simd v0.0.0-20160723061019-3f5f724cb5b1/go.mod h1:pD8RvIylQ358TN4wwqatJ8rNavkEINozVn9DtGI3dfQ= github.com/minio/highwayhash v1.0.2 h1:Aak5U0nElisjDCfPSG79Tgzkn2gl66NxOMspRrKnA/g= github.com/minio/highwayhash v1.0.2/go.mod h1:BQskDq+xkJ12lmlUUi7U0M5Swg3EWR+dLTk+kldvVxY= github.com/minio/sha256-simd v0.1.0/go.mod h1:2FMWW+8GMoPweT6+pI63m9YE3Lmw4J71hV56Chs1E/U= +github.com/minio/sha256-simd v0.1.1-0.20190913151208-6de447530771/go.mod h1:B5e1o+1/KgNmWrSQK08Y6Z1Vb5pwIktudl0J58iy0KM= +github.com/minio/sha256-simd v0.1.1/go.mod h1:B5e1o+1/KgNmWrSQK08Y6Z1Vb5pwIktudl0J58iy0KM= github.com/minio/sha256-simd v1.0.0 h1:v1ta+49hkWZyvaKwrQB8elexRqm6Y0aMLjCNsrYxo6g= github.com/minio/sha256-simd v1.0.0/go.mod h1:OuYzVNI5vcoYIAmbIvHPl3N3jUzVedXbKy5RFepssQM= github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= @@ -564,9 +815,12 @@ github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0Qu github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/mitchellh/mapstructure v1.3.2/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= +github.com/mitchellh/mapstructure v1.4.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= +github.com/mitchellh/mapstructure v1.4.2/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/mitchellh/pointerstructure v1.2.0 h1:O+i9nHnXS3l/9Wu7r4NrEdwA2VFTicjUEN1uBnDo34A= +github.com/mitchellh/pointerstructure v1.2.0/go.mod h1:BRAsLI5zgXmw97Lf6s25bs8ohIXc3tViBH44KcwB2g4= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -575,55 +829,100 @@ github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3Rllmb github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 h1:RWengNIwukTxcDr9M+97sNutRR1RKhG96O6jWumTTnw= github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826/go.mod h1:TaXosZuwdSHYgviHp1DAtfrULt5eUgsSMsZf+YrPgl8= +github.com/mr-tron/base58 v1.1.0/go.mod h1:xcD2VGqlgYjBdcBLw+TuYLr8afG+Hj8g2eTVqeSzSU8= +github.com/mr-tron/base58 v1.1.3/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjWI2mNwc= github.com/mr-tron/base58 v1.2.0 h1:T/HDJBh4ZCPbU39/+c3rRvE0uKBQlU27+QI8LJ4t64o= github.com/mr-tron/base58 v1.2.0/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjWI2mNwc= +github.com/mschoch/smat v0.0.0-20160514031455-90eadee771ae/go.mod h1:qAyveg+e4CE+eKJXWVjKXM4ck2QobLqTDytGJbLLhJg= github.com/mssola/user_agent v0.5.2 h1:CZkTUahjL1+OcZ5zv3kZr8QiJ8jy2H08vZIEkBeRbxo= github.com/mssola/user_agent v0.5.2/go.mod h1:TTPno8LPY3wAIEKRpAtkdMT0f8SE24pLRGPahjCH4uw= +github.com/multiformats/go-base32 v0.0.3/go.mod h1:pLiuGC8y0QR3Ue4Zug5UzK9LjgbkL8NSQj0zQ5Nz/AA= github.com/multiformats/go-base32 v0.1.0 h1:pVx9xoSPqEIQG8o+UbAe7DNi51oej1NtK+aGkbLYxPE= github.com/multiformats/go-base32 v0.1.0/go.mod h1:Kj3tFY6zNr+ABYMqeUNeGvkIC/UYgtWibDcT0rExnbI= +github.com/multiformats/go-base36 v0.1.0/go.mod h1:kFGE83c6s80PklsHO9sRn2NCoffoRdUUOENyW/Vv6sM= github.com/multiformats/go-base36 v0.2.0 h1:lFsAbNOGeKtuKozrtBsAkSVhv1p9D0/qedU9rQyccr0= github.com/multiformats/go-base36 v0.2.0/go.mod h1:qvnKE++v+2MWCfePClUEjE78Z7P2a1UV0xHgWc0hkp4= +github.com/multiformats/go-multiaddr v0.2.2/go.mod h1:NtfXiOtHvghW9KojvtySjH5y0u0xW5UouOmQQrn6a3Y= +github.com/multiformats/go-multiaddr v0.3.2/go.mod h1:lCKNGP1EQ1eZ35Za2wlqnabm9xQkib3fyB+nZXHLag0= +github.com/multiformats/go-multibase v0.0.3/go.mod h1:5+1R4eQrT3PkYZ24C3W2Ue2tPwIdYQD509ZjSb5y9Oc= github.com/multiformats/go-multibase v0.1.1 h1:3ASCDsuLX8+j4kx58qnJ4YFq/JWTJpCyDW27ztsVTOI= github.com/multiformats/go-multibase v0.1.1/go.mod h1:ZEjHE+IsUrgp5mhlEAYjMtZwK1k4haNkcaPg9aoe1a8= +github.com/multiformats/go-multihash v0.0.13/go.mod h1:VdAWLKTwram9oKAatUcLxBNUjdtcVwxObEQBtRfuyjc= +github.com/multiformats/go-multihash v0.0.14/go.mod h1:VdAWLKTwram9oKAatUcLxBNUjdtcVwxObEQBtRfuyjc= +github.com/multiformats/go-multihash v0.0.15/go.mod h1:D6aZrWNLFTV/ynMpKsNtB40mJzmCl4jb1alC0OvHiHg= github.com/multiformats/go-multihash v0.2.1 h1:aem8ZT0VA2nCHHk7bPJ1BjUbHNciqZC/d16Vve9l108= github.com/multiformats/go-multihash v0.2.1/go.mod h1:WxoMcYG85AZVQUyRyo9s4wULvW5qrI9vb2Lt6evduFc= +github.com/multiformats/go-varint v0.0.5/go.mod h1:3Ls8CIEsrijN6+B7PbrXRPxHRPuXSrVKRY101jdMZYE= +github.com/multiformats/go-varint v0.0.6/go.mod h1:3Ls8CIEsrijN6+B7PbrXRPxHRPuXSrVKRY101jdMZYE= github.com/multiformats/go-varint v0.0.7 h1:sWSGR+f/eu5ABZA2ZpYKBILXTTs9JWpdEM/nEGOHFS8= github.com/multiformats/go-varint v0.0.7/go.mod h1:r8PUYw/fD/SjBCiKOoDlGF6QawOELpZAu9eioSos/OU= github.com/mvdan/xurls v1.1.0 h1:OpuDelGQ1R1ueQ6sSryzi6P+1RtBpfQHM8fJwlE45ww= github.com/mvdan/xurls v1.1.0/go.mod h1:tQlNn3BED8bE/15hnSL2HLkDeLWpNPAwtw7wkEq44oU= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= +github.com/naoina/go-stringutil v0.1.0/go.mod h1:XJ2SJL9jCtBh+P9q5btrd/Ylo8XwT/h1USek5+NqSA0= +github.com/naoina/toml v0.1.2-0.20170918210437-9fafd6967416/go.mod h1:NBIhNtsFMo3G2szEBne+bO4gS192HuIYRqfvOWb4i1E= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= github.com/nishanths/predeclared v0.0.0-20200524104333-86fad755b4d3/go.mod h1:nt3d53pc1VYcphSCIaYAJtnPYnr3Zyn8fMq2wvPGPso= github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= +github.com/olekukonko/tablewriter v0.0.1/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo= +github.com/olekukonko/tablewriter v0.0.2-0.20190409134802-7e037d187b0c/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo= +github.com/olekukonko/tablewriter v0.0.4/go.mod h1:zq6QwlOf5SlnkVbMSr5EoBv3636FWnp+qbPhuoO21uA= github.com/olekukonko/tablewriter v0.0.5 h1:P2Ga83D34wi1o9J6Wh1mRuqd4mF/x/lgBS7N7AbDhec= +github.com/olekukonko/tablewriter v0.0.5/go.mod h1:hPp6KlRPjbx+hW8ykQs1w3UBbZlj6HuIJcUGPhkA7kY= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.10.1/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= +github.com/onsi/ginkgo v1.14.0/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY= github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vvnwo0= github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE= github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU= github.com/onsi/ginkgo/v2 v2.1.3/go.mod h1:vw5CSIxN1JObi/U8gcbwft7ZxR2dgaR70JSE3/PpL4c= +github.com/onsi/gomega v1.4.1/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= +github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= +github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= github.com/onsi/gomega v1.17.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAlGdZY= github.com/onsi/gomega v1.19.0 h1:4ieX6qQjPP/BfC3mpsAtIGGlxTWPeA3Inl/7DtXw1tw= github.com/onsi/gomega v1.19.0/go.mod h1:LY+I3pBVzYsTBU1AnDwOSxaYi9WoWiqgwooUqq9yPro= +github.com/openconfig/gnmi v0.0.0-20190823184014-89b2bf29312c/go.mod h1:t+O9It+LKzfOAhKTT5O0ehDix+MTqbtT0T9t+7zzOvc= +github.com/openconfig/reference v0.0.0-20190727015836-8dfd928c9696/go.mod h1:ym2A+zigScwkSEb/cVQB0/ZMpU3rqiH6X7WRRsxgOGw= +github.com/opentracing/opentracing-go v1.0.2/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= +github.com/opentracing/opentracing-go v1.0.3-0.20180606204148-bd9c31933947/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= +github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaRUnok+kx1WdO15EQc= github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ= +github.com/paulbellamy/ratecounter v0.2.0/go.mod h1:Hfx1hDpSGoqxkVVpBi/IlYD7kChlfo5C6hzIHwPqfFE= github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 h1:onHthvaw9LFnH4t2DcNVpwGmV9E1BkGknEliJkfwQj0= github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58/go.mod h1:DXv8WO4yhMYhSNPKjeNKa5WY9YCIEBRbNzFFPJbWO6Y= +github.com/pborman/uuid v0.0.0-20170112150404-1b00554d8222/go.mod h1:VyrYX9gd7irzKovcSS6BIIEwPRkP2Wm2m9ufcdFSJ34= +github.com/pborman/uuid v1.2.1 h1:+ZZIw58t/ozdjRaXh/3awHfmWRbzYxJoAdNJxe/3pvw= +github.com/pborman/uuid v1.2.1/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= +github.com/pelletier/go-toml v1.9.3 h1:zeC5b1GviRUyKYd6OJPvBU/mcVDVoL1OhT17FCt5dSQ= +github.com/pelletier/go-toml v1.9.3/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= +github.com/peterh/liner v1.0.1-0.20180619022028-8c1271fcf47f/go.mod h1:xIteQHvHuaLYG9IFj6mSxM0fCKrs34IrEQUhOYuGPHc= +github.com/peterh/liner v1.1.1-0.20190123174540-a2c9a5303de7/go.mod h1:CRroGNssyjTd/qIG2FyxByd2S8JEAZXBl4qUrZf8GS0= +github.com/peterh/liner v1.2.0/go.mod h1:CRroGNssyjTd/qIG2FyxByd2S8JEAZXBl4qUrZf8GS0= +github.com/peterh/liner v1.2.1/go.mod h1:CRroGNssyjTd/qIG2FyxByd2S8JEAZXBl4qUrZf8GS0= +github.com/philhofer/fwd v1.0.0/go.mod h1:gk3iGcWd9+svBvR0sR+KPcfE+RNWozjowpeBVG3ZVNU= github.com/phyber/negroni-gzip v0.0.0-20180113114010-ef6356a5d029 h1:d6HcSW4ZoNlUWrPyZtBwIu8yv4WAWIU3R/jorwVkFtQ= github.com/phyber/negroni-gzip v0.0.0-20180113114010-ef6356a5d029/go.mod h1:94RTq2fypdZCze25ZEZSjtbAQRT3cL/8EuRUqAZC/+w= +github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= +github.com/pierrec/lz4 v2.4.1+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/profile v1.5.0/go.mod h1:qBsxPvzyUincmltOk6iyRVxHYg4adc0OFOv72ZdLa18= +github.com/pkg/sftp v1.10.1/go.mod h1:lYOWFsE0bwd1+KfKJaKeuokY15vzFx25BLbzYYoAxZI= +github.com/pkg/term v0.0.0-20180730021639-bffc007b7fd5/go.mod h1:eCbImbZ95eXtAUIbLAuAVnBnwf83mjf6QIVH8SHYwqQ= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI= @@ -634,6 +933,7 @@ github.com/prestonvanloon/go v1.1.7-0.20190722034630-4f2e55fcf87b/go.mod h1:KdQU github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso= github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= +github.com/prometheus/client_golang v1.4.1/go.mod h1:e9GMxYsXl05ICDXkRhurwBS4Q3OK1iX/F2sw+iXX5zU= github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M= github.com/prometheus/client_golang v1.11.0/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0= github.com/prometheus/client_golang v1.12.1/go.mod h1:3Z9XVyYiZYEO+YQWt3RD2R3jrbd179Rt297l4aS6nDY= @@ -648,6 +948,8 @@ github.com/prometheus/client_model v0.3.0/go.mod h1:LDGWKZIo7rky3hgvBe+caln+Dr3d github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= +github.com/prometheus/common v0.6.0/go.mod h1:eBmuwkDJBwy6iBfxCBob6t6dR6ENT/y+J+Zk0j9GMYc= +github.com/prometheus/common v0.9.1/go.mod h1:yhUN8i9wzaXS3w1O07YhxHEBxD+W35wd8bs7vj7HSQ4= github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo= github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc= github.com/prometheus/common v0.32.1/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls= @@ -656,18 +958,34 @@ github.com/prometheus/common v0.39.0/go.mod h1:6XBZ7lYdLCbkAVhwRsWTZn+IN5AB9F/NX github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= +github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A= +github.com/prometheus/procfs v0.0.10/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A= github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= github.com/prometheus/procfs v0.9.0 h1:wzCHvIvM5SxWqYvwgVL7yJY8Lz3PKn49KQtpgMYJfhI= github.com/prometheus/procfs v0.9.0/go.mod h1:+pB4zwohETzFnmlpe6yd2lSc+0/46IYZRB/chUwxUZY= +github.com/prometheus/tsdb v0.6.2-0.20190402121629-4f204dcbc150/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= +github.com/prometheus/tsdb v0.10.0/go.mod h1:oi49uRhEe9dPUTlS3JRZOwJuVi6tmh10QSgwXEyGCt4= +github.com/protolambda/bls12-381-util v0.0.0-20210720105258-a772f2aac13e h1:ugvwIKDzqL6ODJciRPMm+9xFQ5AlOYHeMpCOeEuP7LA= +github.com/protolambda/bls12-381-util v0.0.0-20210720105258-a772f2aac13e/go.mod h1:MPZvj2Pr0N8/dXyTPS5REeg2sdLG7t8DRzC1rLv925w= +github.com/protolambda/eth2-val-tools v0.1.1 h1:IiiRtMO6ki6KRCJZuhLx46zeTEmRcufP8J0+7UcpBFk= +github.com/protolambda/eth2-val-tools v0.1.1/go.mod h1:CB/bzTq6ubTBmOIWk/Yga6e3zugcPYZEbiHG1pNHjaI= +github.com/protolambda/eth2-val-tools v0.1.2-0.20230223182759-4bf01453537a h1:d5tJf7+aZVqGnjM60W8QgHzAH88eKR1MjbVya2YuvDk= +github.com/protolambda/eth2-val-tools v0.1.2-0.20230223182759-4bf01453537a/go.mod h1:CB/bzTq6ubTBmOIWk/Yga6e3zugcPYZEbiHG1pNHjaI= +github.com/protolambda/go-keystorev4 v0.0.0-20211007151826-f20444f6d564 h1:yCXGkFjrZ8EggxW+Y7ueRZesNcBk0avLU0mVU/I2KtU= +github.com/protolambda/go-keystorev4 v0.0.0-20211007151826-f20444f6d564/go.mod h1:Xda3KO8+DMyWaTr+LwUUpVRTB5SdFzoKu0ivXNI6p1s= github.com/protolambda/messagediff v1.4.0/go.mod h1:LboJp0EwIbJsePYpzh5Op/9G1/4mIztMRYzzwR0dR2M= github.com/protolambda/zrnt v0.12.4 h1:BJE8HSYx9wJnOOBr/QdYCE0fxUOfJpSKbSCcXGprm28= github.com/protolambda/zrnt v0.12.4/go.mod h1:7G3u3+BgUjmQw75fQ3OYOJucujRrspBmshCx3OdRutc= +github.com/protolambda/zrnt v0.30.0 h1:pHEn69ZgaDFGpLGGYG1oD7DvYI7RDirbMBPfbC+8p4g= +github.com/protolambda/zrnt v0.30.0/go.mod h1:qcdX9CXFeVNCQK/q0nswpzhd+31RHMk2Ax/2lMsJ4Jw= github.com/protolambda/zssz v0.1.5 h1:7fjJjissZIIaa2QcvmhS/pZISMX21zVITt49sW1ouek= github.com/protolambda/zssz v0.1.5/go.mod h1:a4iwOX5FE7/JkKA+J/PH0Mjo9oXftN6P8NZyL28gpag= github.com/protolambda/ztyp v0.1.0/go.mod h1:u9yT4ioIokwlHrOfiZ52ezHfKdza3phxhTJix01vGgU= +github.com/protolambda/ztyp v0.2.2 h1:rVcL3vBu9W/aV646zF6caLS/dyn9BN8NYiuJzicLNyY= +github.com/protolambda/ztyp v0.2.2/go.mod h1:9bYgKGqg3wJqT9ac1gI2hnVb0STQq7p/1lapqrqY1dU= github.com/prysmaticlabs/fastssz v0.0.0-20221107182844-78142813af44 h1:c3p3UzV4vFA7xaCDphnDWOjpxcadrQ26l5b+ypsvyxo= github.com/prysmaticlabs/fastssz v0.0.0-20221107182844-78142813af44/go.mod h1:MA5zShstUwCQaE9faGHgCGvEWUbG87p4SAXINhmCkvg= github.com/prysmaticlabs/go-bitfield v0.0.0-20210809151128-385d8c5e3fb7 h1:0tVE4tdWQK9ZpYygoV7+vS6QkDvQVySboMVEIxBJmXw= @@ -682,8 +1000,13 @@ github.com/prysmaticlabs/grpc-gateway/v2 v2.3.1-0.20210702154020-550e1cd83ec1/go github.com/prysmaticlabs/protoc-gen-go-cast v0.0.0-20211014160335-757fae4f38c6 h1:+jhXLjEYVW4qU2z5SOxlxN+Hv/A9FDf0HpfDurfMEz0= github.com/r3labs/sse/v2 v2.7.4 h1:pvCMswPDlXd/ZUFx1dry0LbXJNHXwWPulLcUGYwClc0= github.com/r3labs/sse/v2 v2.7.4/go.mod h1:hUrYMKfu9WquG9MyI0r6TKiNH+6Sw/QPKm2YbNbU5g8= +github.com/rcrowley/go-metrics v0.0.0-20190826022208-cac0b30c2563/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE= +github.com/retailnext/hllpp v1.0.1-0.20180308014038-101a6d2f8b52/go.mod h1:RDpi1RftBQPUCDRw6SmxeaREsAaRKnOclghuzp/WRzc= +github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= github.com/rivo/uniseg v0.4.3 h1:utMvzDsuh3suAEnhH0RdHmoPbU648o6CvXxTx4SBMOw= +github.com/rjeczalik/notify v0.9.1/go.mod h1:rKwnCoCGeuQnwBtTSPL9Dad03Vh2n40ePRrjvIXnJho= +github.com/rjeczalik/notify v0.9.2/go.mod h1:aErll2f0sUX9PXZnVNyeiObbmTlk5jnMoCa4QEjJeqM= github.com/rocket-pool/go-merkletree v1.0.1-0.20220406020931-c262d9b976dd h1:p9KuetSKB9nte9I/MkkiM3pwKFVQgqxxPTQ0y56Ff6s= github.com/rocket-pool/go-merkletree v1.0.1-0.20220406020931-c262d9b976dd/go.mod h1:UE9fof8P7iESVtLn1K9CTSkNRYVFHZHlf96RKbU33kA= github.com/rocket-pool/rocketpool-go v1.10.1-0.20230228020137-d5a680907dff h1:rOZevts77yp6t7J9xPRxhM1s4xyp4vbfqQJlFIWdDGE= @@ -694,8 +1017,11 @@ github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6So github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8= +github.com/rs/cors v0.0.0-20160617231935-a62a804a8a00/go.mod h1:gFx+x8UowdsKA9AchylcLynDq+nNFfI8FkUZdN/jGCU= +github.com/rs/cors v1.7.0/go.mod h1:gFx+x8UowdsKA9AchylcLynDq+nNFfI8FkUZdN/jGCU= github.com/rs/cors v1.8.0 h1:P2KMzcFwrPoSjkF1WLRPsp3UMLyql8L4v9hQpVeK5so= github.com/rs/cors v1.8.0/go.mod h1:EBwu+T5AvHOcXwvZIkQFjUN6s8Czyqw12GL/Y0tUyRM= +github.com/rs/xhandler v0.0.0-20160618193221-ed27b6fd6521/go.mod h1:RvLn4FgxWubrpZHtQLnOf6EwhN2hEMusxZOhcW9H3UQ= github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ= github.com/rs/xid v1.3.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= github.com/rs/zerolog v1.13.0/go.mod h1:YbFCdg8HfsridGWAh22vktObvhZbQsZXe4/zB0OKkWU= @@ -708,6 +1034,14 @@ github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQD github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= +github.com/segmentio/kafka-go v0.1.0/go.mod h1:X6itGqS9L4jDletMsxZ7Dz+JFWxM6JHfPOCvTvk+EJo= +github.com/segmentio/kafka-go v0.2.0/go.mod h1:X6itGqS9L4jDletMsxZ7Dz+JFWxM6JHfPOCvTvk+EJo= +github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= +github.com/shirou/gopsutil v2.20.5+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA= +github.com/shirou/gopsutil v2.20.9+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA= +github.com/shirou/gopsutil v3.21.4-0.20210419000835-c7a38de76ee5+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA= +github.com/shirou/gopsutil v3.21.5+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA= +github.com/shirou/gopsutil v3.21.7+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA= github.com/shirou/gopsutil v3.21.11+incompatible h1:+1+c1VGhc88SSonWP6foOcLhvnKlUeu/erjjvaPEYiI= github.com/shirou/gopsutil v3.21.11+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA= github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24/go.mod h1:M+9NzErvs504Cn4c5DxATwIqPbtswREoFCre64PpcG4= @@ -719,6 +1053,7 @@ github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPx github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= +github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/sirupsen/logrus v1.9.0 h1:trlNQbNUG3OdDrDil03MCb1H2o9nJ1x4/5LYw7byDE0= github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e h1:MRM5ITcdelLK2j1vwZ3Je0FKVCfqOLp5zO6trqMLYs0= @@ -730,18 +1065,38 @@ github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasO github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI= github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= +github.com/spf13/afero v1.6.0 h1:xoax2sJ2DT8S8xA2paPFjDCScCNeWsg75VG0DLRreiY= +github.com/spf13/afero v1.6.0/go.mod h1:Ai8FlHk4v/PARR026UzYexafAt9roJ7LcLMAmO6Z93I= github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= +github.com/spf13/cast v1.3.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= +github.com/spf13/cast v1.4.0 h1:WhlbjwB9EGCc8W5Rxdkus+wmH2ASRwwTJk6tgHKwdqQ= +github.com/spf13/cast v1.4.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= +github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= github.com/spf13/cobra v1.0.1-0.20201006035406-b97b5ead31f7/go.mod h1:yk5b0mALVusDL5fMM6Rd1wgnoO5jUPhwsQ6LQAJTidQ= +github.com/spf13/cobra v1.1.3 h1:xghbfqPkxzxP3C/f3n5DdpAbdKLj4ZE4BWQI362l53M= +github.com/spf13/cobra v1.1.3/go.mod h1:pGADOWyqRD/YMrPZigI/zbliZ2wVD/23d+is3pSWzOo= +github.com/spf13/cobra v1.2.1 h1:+KmjbUw1hriSNMF55oPrkZcb27aECyrj8V2ytv7kWDw= +github.com/spf13/cobra v1.2.1/go.mod h1:ExllRjgxM/piMAM+3tAZvg8fsklGAf3tPfi+i8t68Nk= github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= +github.com/spf13/jwalterweatherman v1.1.0 h1:ue6voC5bR5F8YxI5S67j9i582FU4Qvo2bmqnqMYADFk= +github.com/spf13/jwalterweatherman v1.1.0/go.mod h1:aNWZUN0dPAAO/Ljvb5BEdw96iTZ0EXowPYD95IqWIGo= github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= +github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/spf13/viper v1.7.0/go.mod h1:8WkrPz2fc9jxqZNCJI/76HCieCp4Q8HaLFoCha5qpdg= +github.com/spf13/viper v1.8.1 h1:Kq1fyeebqsBfbjZj4EL7gj2IO0mMaiyjYUWcUsl2O44= +github.com/spf13/viper v1.8.1/go.mod h1:o0Pch8wJ9BVSWGQMbra6iw0oQ5oktSIBaujf1rJH9Ns= +github.com/status-im/keycard-go v0.0.0-20190316090335-8537d3370df4/go.mod h1:RZLeN1LMWmRsyYjvAu+I6Dm9QmlDaIIt+Y+4Kd7Tp+Q= +github.com/status-im/keycard-go v0.0.0-20200402102358-957c09536969/go.mod h1:RZLeN1LMWmRsyYjvAu+I6Dm9QmlDaIIt+Y+4Kd7Tp+Q= github.com/status-im/keycard-go v0.2.0 h1:QDLFswOQu1r5jsycloeQh3bVU8n/NatHHaZobtDnDzA= +github.com/steakknife/bloomfilter v0.0.0-20180922174646-6819c0d2a570/go.mod h1:8OR4w3TdeIHIh1g6EMY5p0gVNOovcWC+1vpc7naMuAw= +github.com/steakknife/hamming v0.0.0-20180906055917-c99c65617cd3/go.mod h1:hpGUWaI9xL8pRQCTXQgocU38Qw1g0Us7n5PxxTwTCYU= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/testify v1.2.0/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= @@ -755,6 +1110,7 @@ github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8= github.com/stripe/stripe-go/v72 v72.50.0 h1:oy+EsSKMrFS3zzayb8Ic+2LZ04Ux0vJ4990/7psaYsc= github.com/stripe/stripe-go/v72 v72.50.0/go.mod h1:QwqJQtduHubZht9mek5sds9CtQcKFdsykV9ZepRWwo0= +github.com/subosito/gotenv v1.2.0 h1:Slr1R9HxAlEKefgq5jn9U+DnETlIUa6HfgEzj0g5d7s= github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= github.com/supranational/blst v0.3.10 h1:CMciDZ/h4pXDDXQASe8ZGTNKUiVNxVVA5hpci2Uuhuk= github.com/supranational/blst v0.3.10/go.mod h1:jZJtfjgudtNl4en1tzwPIV3KjUnQUvG3/j+w+fVonLw= @@ -764,43 +1120,81 @@ github.com/swaggo/http-swagger v1.3.0 h1:1+6M4qRorIbdyTWTsGrwnb0r9jGK5dcWN82O6oY github.com/swaggo/http-swagger v1.3.0/go.mod h1:9glekdg40lwclrrKNRGgj/IMDxpNPZ3kzab4oPcF8EM= github.com/swaggo/swag v1.8.3 h1:3pZSSCQ//gAH88lfmxM3Cd1+JCsxV8Md6f36b9hrZ5s= github.com/swaggo/swag v1.8.3/go.mod h1:jMLeXOOmYyjk8PvHTsXBdrubsNd9gUJTTCzL5iBnseg= +github.com/syndtr/goleveldb v1.0.1-0.20200815110645-5c35d600f0ca/go.mod h1:u2MKkTVTVJWe5D1rCvame8WqhBd88EuIwODJZ1VHCPM= +github.com/syndtr/goleveldb v1.0.1-0.20210305035536-64b5b1c73954/go.mod h1:u2MKkTVTVJWe5D1rCvame8WqhBd88EuIwODJZ1VHCPM= +github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7/go.mod h1:q4W45IWZaF22tdD+VEXcAWRA037jwmWEB5VWYORlTpc= github.com/syndtr/goleveldb v1.0.1-0.20220721030215-126854af5e6d h1:vfofYNRScrDdvS342BElfbETmL1Aiz3i2t0zfRj16Hs= github.com/syndtr/goleveldb v1.0.1-0.20220721030215-126854af5e6d/go.mod h1:RRCYJbIwD5jmqPI9XoAFR0OcDxqUctll6zUj/+B4S48= +github.com/templexxx/cpufeat v0.0.0-20180724012125-cef66df7f161/go.mod h1:wM7WEvslTq+iOEAMDLSzhVuOt5BRZ05WirO+b09GHQU= +github.com/templexxx/xor v0.0.0-20191217153810-f85b25db303b/go.mod h1:5XA7W9S6mni3h5uvOC75dA3m9CCCaS83lltmc0ukdi4= github.com/thomaso-mirodin/intmath v0.0.0-20160323211736-5dc6d854e46e h1:cR8/SYRgyQCt5cNCMniB/ZScMkhI9nk8U5C7SbISXjo= github.com/thomaso-mirodin/intmath v0.0.0-20160323211736-5dc6d854e46e/go.mod h1:Tu4lItkATkonrYuvtVjG0/rhy15qrNGNTjPdaphtZ/8= +github.com/tinylib/msgp v1.0.2/go.mod h1:+d+yLhGm8mzTaHzB+wgMYrodPfmZrzkirds8fDWklFE= +github.com/tjfoc/gmsm v1.3.0/go.mod h1:HaUcFuY0auTiaHB9MHFGCPx5IaLhTUd2atbCFBQXn9w= +github.com/tklauser/go-sysconf v0.3.5/go.mod h1:MkWzOF4RMCshBAMXuhXJs64Rte09mITnppBXY/rYEFI= +github.com/tklauser/go-sysconf v0.3.6/go.mod h1:MkWzOF4RMCshBAMXuhXJs64Rte09mITnppBXY/rYEFI= +github.com/tklauser/go-sysconf v0.3.7/go.mod h1:JZIdXh4RmBvZDBZ41ld2bGxRV3n4daiiqA3skYhAoQ4= github.com/tklauser/go-sysconf v0.3.11 h1:89WgdJhk5SNwJfu+GKyYveZ4IaJ7xAkecBo+KdJV0CM= github.com/tklauser/go-sysconf v0.3.11/go.mod h1:GqXfhXY3kiPa0nAXPDIQIWzJbMCB7AmcWpGR8lSZfqI= +github.com/tklauser/numcpus v0.2.2/go.mod h1:x3qojaO3uyYt0i56EW/VUYs7uBvdl2fkfZFu0T9wgjM= +github.com/tklauser/numcpus v0.2.3/go.mod h1:vpEPS/JC+oZGGQ/My/vJnNsvMDQL6PwOqt8dsCw5j+E= github.com/tklauser/numcpus v0.6.0 h1:kebhY2Qt+3U6RNK7UqpYNA+tJ23IBEGKkB7JQBfDYms= github.com/tklauser/numcpus v0.6.0/go.mod h1:FEZLMke0lhOUG6w2JadTzp0a+Nl8PF/GFkQ5UVIcaL4= github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/twitchtv/twirp v7.1.0+incompatible/go.mod h1:RRJoFSAmTEh2weEqWtpPE3vFK5YBhA6bqp2l1kfCC5A= +github.com/tyler-smith/go-bip39 v1.0.1-0.20181017060643-dbb3b84ba2ef/go.mod h1:sJ5fKU0s6JVwZjjcUEX2zFOnvq0ASQ2K9Zr6cf67kNs= +github.com/tyler-smith/go-bip39 v1.0.2/go.mod h1:sJ5fKU0s6JVwZjjcUEX2zFOnvq0ASQ2K9Zr6cf67kNs= github.com/tyler-smith/go-bip39 v1.1.0 h1:5eUemwrMargf3BSLRRCalXT93Ns6pQJIjYQN2nyfOP8= +github.com/tyler-smith/go-bip39 v1.1.0/go.mod h1:gUYDtqQw1JS3ZJ8UWVcGTGqqr6YIN3CWg+kkNaLt55U= github.com/uber/jaeger-client-go v2.25.0+incompatible h1:IxcNZ7WRY1Y3G4poYlx24szfsn/3LvK9QHCq9oQw8+U= github.com/ugorji/go v1.1.7/go.mod h1:kZn38zHttfInRq0xu/PH0az30d+z6vm202qpg1oXVMw= github.com/ugorji/go/codec v1.1.7/go.mod h1:Ax+UKWsSmolVDwsd+7N3ZtXu+yMGCf907BLYF3GoBXY= github.com/umbracle/gohashtree v0.0.2-alpha.0.20230207094856-5b775a815c10 h1:CQh33pStIp/E30b7TxDlXfM0145bn2e8boI30IxAhTg= +github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= github.com/urfave/cli v1.22.12 h1:igJgVw1JdKH+trcLWLeLwZjU9fEfPesQ+9/e4MQ44S8= github.com/urfave/cli v1.22.12/go.mod h1:sSBEIC79qR6OvcmsD4U3KABeOTxDqQtdDnaFuUN30b8= +github.com/urfave/cli/v2 v2.3.0/go.mod h1:LJmUH05zAU44vOAcrfzZQKsZbVcdbOG8rtL3/XcUArI= github.com/urfave/cli/v2 v2.17.2-0.20221006022127-8f469abc00aa h1:5SqCsI/2Qya2bCzK15ozrqo2sZxkh0FHynJZOTVoV6Q= github.com/urfave/negroni v1.0.0 h1:kIimOitoypq34K7TG7DUaJ9kq/N4Ofuwi1sjz0KipXc= github.com/urfave/negroni v1.0.0/go.mod h1:Meg73S6kFm/4PpbYdq35yYWoCZ9mS/YSx+lKnmiohz4= +github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= +github.com/valyala/fasttemplate v1.0.1/go.mod h1:UQGH1tvbgY+Nz5t2n7tXsz52dQxojPUpymEIMZ47gx8= +github.com/valyala/fasttemplate v1.2.1/go.mod h1:KHLXt3tVN2HBp8eijSv/kGJopbvo7S+qRAEEKiv+SiQ= +github.com/wealdtech/ethereal v0.0.0-20211110093529-ed2beb2b4a38 h1:k5dfyU/CXesxiko9wpZN05blJUIax8mMXGVS7Uk2r/Q= +github.com/wealdtech/ethereal v0.0.0-20211110093529-ed2beb2b4a38/go.mod h1:Gj3SOtZtee/3+3psis6VLuXzGZDWKH0mxorvvJSIjes= +github.com/wealdtech/go-bytesutil v1.1.1/go.mod h1:jENeMqeTEU8FNZyDFRVc7KqBdRKSnJ9CCh26TcuNb9s= github.com/wealdtech/go-bytesutil v1.2.1 h1:TjuRzcG5KaPwaR5JB7L/OgJqMQWvlrblA1n0GfcXFSY= github.com/wealdtech/go-bytesutil v1.2.1/go.mod h1:RhUDUGT1F4UP4ydqbYp2MWJbAel3M+mKd057Pad7oag= +github.com/wealdtech/go-ens/v3 v3.5.1/go.mod h1:bVuYoWYEEeEu7Zy95rIMjPR34QFJarxt8p84ywSo0YM= github.com/wealdtech/go-ens/v3 v3.5.5 h1:/jq3CDItK0AsFnZtiFJK44JthkAMD5YE3WAJOh4i7lc= github.com/wealdtech/go-ens/v3 v3.5.5/go.mod h1:w0EDKIm0dIQnqEKls6ORat/or+AVfPEdEXVfN71EeEE= +github.com/wealdtech/go-erc1820 v1.2.3 h1:BmA93UGhZhAKN1AJoZ74JyQ+N93CBgNIsD8MZUjGbWY= +github.com/wealdtech/go-erc1820 v1.2.3/go.mod h1:iAQx1hyjt3165fwe/2oTUmysKQYZSJ46HkMPsaRxVew= +github.com/wealdtech/go-eth2-types/v2 v2.5.3/go.mod h1:JCIc6C8SAaxQPasY2q3g7iFUuebmKURb+TyqCKrmq4c= +github.com/wealdtech/go-eth2-types/v2 v2.6.0/go.mod h1:psOez/ZRBzZSDl5hiNDwRf5ZqQujNE6h5FxAz09Koxg= github.com/wealdtech/go-eth2-types/v2 v2.8.1 h1:y2N3xSIZ3tVqsnvj4AgPkh48U5sM612vhZwlK3k+3lM= github.com/wealdtech/go-eth2-types/v2 v2.8.1/go.mod h1:3TJShI4oBzG8pCZsfe3NZAq8QAmXrC2rd45q7Vn/XB8= +github.com/wealdtech/go-eth2-util v1.6.4/go.mod h1:OB2m2u3kO3yVqJA3v1jVaWH1Stv5cBmAlBFX47cokRI= github.com/wealdtech/go-eth2-util v1.8.1 h1:nb50hygsNoql94akg7GN6im/weg8ZZgJWHgiyrj8qiU= github.com/wealdtech/go-eth2-util v1.8.1/go.mod h1:vv+8jVgYRXEGty/VLPNn1RYlbQNYmTht3VR6nfh0z4E= github.com/wealdtech/go-multicodec v1.4.0 h1:iq5PgxwssxnXGGPTIK1srvt6U5bJwIp7k6kBrudIWxg= github.com/wealdtech/go-multicodec v1.4.0/go.mod h1:aedGMaTeYkIqi/KCPre1ho5rTb3hGpu/snBOS3GQLw4= github.com/wealdtech/go-string2eth v1.1.0 h1:USJQmysUrBYYmZs7d45pMb90hRSyEwizP7lZaOZLDAw= +github.com/wealdtech/go-string2eth v1.1.0/go.mod h1:RUzsLjJtbZaJ/3UKn9kY19a/vCCUHtEWoUW3uiK6yGU= +github.com/willf/bitset v1.1.3/go.mod h1:RjeCKbqT1RxIR/KWY6phxZiaY1IyutSBfGjNPySAYV4= +github.com/wsddn/go-ecdh v0.0.0-20161211032359-48726bab9208/go.mod h1:IotVbo4F+mw0EzQ08zFqg7pK3FebNXpaMsRy2RT+Ees= +github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c/go.mod h1:lB8K/P019DLNhemzwFU4jHLhdvlE6uDZjXFejJXr49I= +github.com/xdg/stringprep v1.0.0/go.mod h1:Jhud4/sHMO4oL310DaZAKk9ZaJ08SJfe+sJh0HrGL1Y= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= +github.com/xlab/treeprint v0.0.0-20180616005107-d6fb6747feb6/go.mod h1:ce1O1j6UtZfjr22oyGxGLbauSBp2YVXpARAosm7dHBg= github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 h1:bAn7/zixMGCfxrRTfdpNzjtPYqr8smhKouy9mxVdGPU= +github.com/xtaci/kcp-go v5.4.20+incompatible/go.mod h1:bN6vIwHQbfHaHtFpEssmWsN45a+AZwO7eyRCmEIbtvE= +github.com/xtaci/lossyconn v0.0.0-20190602105132-8df528c0c9ae/go.mod h1:gXtu8J62kEgmN++bm9BVICuT/e8yiLI2KFobd/TRFsE= github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/yuin/goldmark v1.4.0/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= github.com/yusufpapurcu/wmi v1.2.2 h1:KBNDSne4vP5mbSWnJbO+51IMOXJB67QiYCSBrubbPRg= @@ -809,6 +1203,9 @@ github.com/zenazn/goji v0.9.0/go.mod h1:7S9M489iMyHBNxwZnk9/EHS098H4/F6TATF2mIxt github.com/zesik/proxyaddr v0.0.0-20161218060608-ec32c535184d h1:Gsw/uTjNB2vkIEhBO3NAXjKo6QRY6D5B0GzMv980ses= github.com/zesik/proxyaddr v0.0.0-20161218060608-ec32c535184d/go.mod h1:PXKs8dwJ+2BKBQxMlKg+eaO27VucEN/jrrymnRB5tvs= go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= +go.etcd.io/etcd/api/v3 v3.5.0/go.mod h1:cbVKeC6lCfl7j/8jBhAK6aIYO9XOjdptoxU/nLQcPvs= +go.etcd.io/etcd/client/pkg/v3 v3.5.0/go.mod h1:IJHfcCEKxYu1Os13ZdwCwIUTUVGYTSAM3YSwc9/Ac1g= +go.etcd.io/etcd/client/v2 v2.305.0/go.mod h1:h9puh54ZTgAKtEbut2oe9P4L/oqKCVB6xsXlzd7alYQ= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= @@ -816,6 +1213,7 @@ go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= go.opencensus.io v0.22.6/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E= +go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E= go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= @@ -833,26 +1231,45 @@ go.uber.org/zap v1.9.1/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= go.uber.org/zap v1.13.0/go.mod h1:zwrFLgMcdUuIBviXEYEH1YKNaOBnKXsx2IPda5bBwHM= go.uber.org/zap v1.16.0/go.mod h1:MA8QOfq0BHJwdXa996Y4dYkAqRKB8/1K1QMMZVaNZjQ= +go.uber.org/zap v1.17.0/go.mod h1:MXVU+bhUf/A7Xi2HNOnopQOrmycQ5Ih87HtOu4q5SSo= +golang.org/x/crypto v0.0.0-20170930174604-9419663f5a44/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190411191339-88737f569e3a/go.mod h1:WFFai1msRO1wXaEeE5yQxYXgSfI8pQAWXbQop6sCtWE= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190909091759-094676da4a83/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20191219195013-becbf705a915/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20200115085410-6d4e4cb37c7d/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20200204104054-c9f3fb736b72/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20200221231518-2aa609cf4a9d/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20200510223506-06a226fb4e37/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20200820211705-5c72a883971a/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20201016220609-9e8e0b390897/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20201203163018-be400aefbc4c/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= +golang.org/x/crypto v0.0.0-20201221181555-eec23a3978ad/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= +golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= +golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= golang.org/x/crypto v0.0.0-20210616213533-5ff15b29337e/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.0.0-20211108221036-ceb1ce70b4fa/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20211215165025-cf75a172585e/go.mod h1:P+XmwS30IXTQdn5tA2iutPOUgjI07+tq3H3K9MVA1s8= golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58= golang.org/x/crypto v0.7.0 h1:AvwMYaRytfdeVt3u6mLaxYtErKYjxA2OXjJ1HHq6t3A= golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU= +golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20180807140117-3d87b88a115f/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190125153040-c74c464bbbf2/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= +golang.org/x/exp v0.0.0-20190731235908-ec7cb31e5a56/go.mod h1:JhuoJpWY28nO4Vef9tZUw9qufEGTyX1+7lmHxV5q5G4= golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek= golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY= golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= @@ -862,6 +1279,7 @@ golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EH golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= golang.org/x/exp v0.0.0-20230206171751-46f607a40771 h1:xP7rWLUr1e1n2xkK5YB4LI0hPEy3LJC6Wk+D4pGlOJg= golang.org/x/exp v0.0.0-20230206171751-46f607a40771/go.mod h1:CxIveKay+FTh1D0yPZemJVgC/95VzuuOLq5Qi4xnoYc= +golang.org/x/image v0.0.0-20180708004352-c73c2afc3b81/go.mod h1:ux5Hcp/YLpHSI86hEcLt0YII63i6oz57MZXIpbrjZUs= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= @@ -875,21 +1293,28 @@ golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRu golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/lint v0.0.0-20210508222113-6edffad5e616/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= +golang.org/x/mobile v0.0.0-20200801112145-973feb4309de/go.mod h1:skQtrUTUwhdJvXM/2KKJzY8pDgNr9I/FOMqDVRPBUS4= golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= +golang.org/x/mod v0.1.1-0.20191209134235-331c550502dd/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.9.0 h1:KENHtAZL2y3NLMYZeHY9DW8HW8V+kQyJsY/V9JlKvCs= +golang.org/x/mod v0.9.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/net v0.0.0-20180719180050-a680a1efc54d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181011144130-49bb7cea24b1/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -921,17 +1346,23 @@ golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/ golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20200813134508-3edf25e44fcc/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20200927032502-5d4f70055728/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201026091529-146b70c837a4/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201031054903-ff519b6c9102/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201209123823-ac852fbbde11/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210220033124-5f55cee0dc0d/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4/go.mod h1:RBQZq4jEuRlivfhVLdyRGr576XBO4/greRjx4P4O3yc= +golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk= golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20210614182718-04defd469f4e/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20210726213435-c6fcb2dbf985/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= @@ -948,6 +1379,10 @@ golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4Iltr golang.org/x/oauth2 v0.0.0-20200902213428-5d25da1a8d43/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20201109201403-9fd604954f58/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20201208152858-08078c50e5b5/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210218202405-ba52d332ba99/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210220000619-9bb904979d93/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210313182246-cd4f82c27b84/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210402161424-2e8d93401602/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20210427180440-81ed05c6b58c/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.3.0 h1:6l90koy8/LaBLmLu8jpHeHexzMwEita0zFfYlggy2F8= @@ -970,6 +1405,7 @@ golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180926160741-c2ed4eda69e7/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181026203630-95b1ffbd15a5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -995,11 +1431,13 @@ golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200107162124-548cf772de50/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200219091948-cb0a6d8edb6c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -1008,25 +1446,45 @@ golang.org/x/sys v0.0.0-20200501052902-10377860bb8e/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200501145240-bc7a7d42d5c3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200814200057-3d37ad5750ed/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200824131525-c12d262b63d8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200826173525-f9321e4c35a6/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201026173827-119d4633e4d1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201101102859-da207088b7d1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201214095126-aec9a390925b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210104204734-6f8348627aad/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210220050731-9a76102bfb43/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210303074136-134d130e1a04/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210305230114-8fe3ee5dd75b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210309074719-68d13333faf2/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210315160823-c6e025ad8005/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210316164454-77fc1eacc6aa/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210324051608-47abb6519492/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210420205809-ac73e9fd8988/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210816183151-1e6c022a8912/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211109184856-51b60fd695b3/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220406163625-3f8b81556e12/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -1062,10 +1520,15 @@ golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20201208040808-7e3f01d25324/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20220922220347-f3bd1da661af h1:Yx9k8YCG3dvF87UAn2tu2HQLf2dt/eR1bXxpLMWeH+Y= golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20180525024113-a5b4c53f6e8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190206041539-40960b6deb8e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= @@ -1093,12 +1556,15 @@ golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtn golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200103221440-774c71fcf114/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200108203644-89082a384178/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200117012304-6edc0a871e69/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200117161641-43d50277825c/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200122220014-bf1340f18c4a/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200204074204-1cc6d1ef6c74/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200221224223-e1da425f72fd/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200227222343-706bc42d1f0d/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= @@ -1109,6 +1575,7 @@ golang.org/x/tools v0.0.0-20200512131952-2bc93b1c0c88/go.mod h1:EkVYQZoAsY45+roY golang.org/x/tools v0.0.0-20200515010526-7d3b6ebf133d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200522201501-cb1345f3a375/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200717024301-6ddee64345a6/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= @@ -1119,7 +1586,11 @@ golang.org/x/tools v0.0.0-20201201161351-ac6f37ff4c2a/go.mod h1:emZCQorbCU4vsT4f golang.org/x/tools v0.0.0-20201202200335-bef1c476418a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20201208233053-a543418bbed2/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20210105154028-b0ab187a4818/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= +golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.1.6-0.20210726203631-07bc1bf47fb2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.7/go.mod h1:LGqMHiF4EqQNHR1JncWGqT5BVaXmza+X+BDGol+dOxo= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/tools v0.7.0 h1:W4OVu8VVOaIO0yzWMNdepAulS7YfoS3Zabrm8DOXXU4= @@ -1133,8 +1604,14 @@ golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20220517211312-f3a8303e98df/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8= golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 h1:H2TDz8ibqkAF6YGhCdN3jS9O0/s90v0rJh3X/OLHEUk= golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8= +gonum.org/v1/gonum v0.0.0-20180816165407-929014505bf4/go.mod h1:Y+Yx5eoAFn32cQvJDxZx5Dpnq+c3wtXuadVZAcxbbBo= +gonum.org/v1/gonum v0.0.0-20181121035319-3f7ecaa7e8ca/go.mod h1:Y+Yx5eoAFn32cQvJDxZx5Dpnq+c3wtXuadVZAcxbbBo= +gonum.org/v1/gonum v0.6.0/go.mod h1:9mxDZsDKxgMAuccQkewq682L+0eCu4dCN2yonUJTCLU= gonum.org/v1/gonum v0.12.0 h1:xKuo6hzt+gMav00meVPUlXwSdoEJP46BR+wdxQEFK2o= gonum.org/v1/gonum v0.12.0/go.mod h1:73TDxJfAAHeA8Mk9mf8NlIppyhQNo5GLTcYeqgo2lvY= +gonum.org/v1/netlib v0.0.0-20181029234149-ec6d1f5cefe6/go.mod h1:wa6Ws7BG/ESfp6dHfk7C6KdzKA7wR7u/rKwOGE66zvw= +gonum.org/v1/netlib v0.0.0-20190313105609-8cb42192e0e0/go.mod h1:wa6Ws7BG/ESfp6dHfk7C6KdzKA7wR7u/rKwOGE66zvw= +gonum.org/v1/plot v0.0.0-20190515093506-e2840ee46a6b/go.mod h1:Wt8AAjI+ypCyYX3nZBvf6cAIx93T+c/OS2HFAYskSZc= google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= @@ -1153,6 +1630,10 @@ google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSr google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc= google.golang.org/api v0.35.0/go.mod h1:/XrVsuzM0rZmrsbjJutiuftIzeuTQcEeaYcSk/mQ1dg= google.golang.org/api v0.36.0/go.mod h1:+z5ficQTmoYpPn8LCUNVpK5I7hwkpjbcgqA7I34qYtE= +google.golang.org/api v0.40.0/go.mod h1:fYKFpnQN0DsDSKRVRcQSDQNtqWPfM9i+zNPxepjRCQ8= +google.golang.org/api v0.41.0/go.mod h1:RkxM5lITDfTzmyKFPt+wGrCJbVfniCr2ool8kTBzRTU= +google.golang.org/api v0.43.0/go.mod h1:nQsDGjRXMo4lvh5hP0TKqF244gqhGcr/YSIykhUk/94= +google.golang.org/api v0.44.0/go.mod h1:EBOGZqzyhtvMDoxwS97ctnh0zUmYY6CxqXsc1AvkYD8= google.golang.org/api v0.102.0 h1:JxJl2qQ85fRMPNvlZY/enexbxpCjLwGhZUtgfGeQ51I= google.golang.org/api v0.102.0/go.mod h1:3VFl6/fzoA+qNuS1N1/VfXY4LjoXN/wzeIp7TweWwGo= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= @@ -1168,6 +1649,7 @@ google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRn google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190716160619-c506a9f90610/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= @@ -1175,10 +1657,12 @@ google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvx google.golang.org/genproto v0.0.0-20191115194625-c23dd37a84c9/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= google.golang.org/genproto v0.0.0-20191216164720-4f79533eabd1/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200108215221-bd8f9a0ef82f/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= google.golang.org/genproto v0.0.0-20200115191322-ca5a22157cba/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= google.golang.org/genproto v0.0.0-20200122232147-0452cf42e150/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= google.golang.org/genproto v0.0.0-20200204135345-fa8e72b47b90/go.mod h1:GmwEX6Z4W5gMy59cAlVYjN9JhxgbQH6Gn+gFDQe2lzA= google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200218151345-dad8c97a84f5/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200224152610-e50cd9704f63/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200228133532-8c2c7df3a383/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200305110556-506484158171/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= @@ -1199,8 +1683,15 @@ google.golang.org/genproto v0.0.0-20201201144952-b05cb90ed32e/go.mod h1:FWY/as6D google.golang.org/genproto v0.0.0-20201203001206-6486ece9c497/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20201210142538-e3217bee35cc/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20201211151036-40ec1c210f7a/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201214200347-8c77b98c765d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210207032614-bba0dbe2a9ea/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210222152913-aa3ee6e6a81c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210303154014-9728d6b83eeb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210310155132-4ce2db91004e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210319143718-93e7006c17a6/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210402141018-6c239bbf2bb1/go.mod h1:9lPAdzaEmUacj36I+k7YKbEc5CXzPIeORRgDAUOu28A= google.golang.org/genproto v0.0.0-20210426193834-eac7f76ac494/go.mod h1:P3QM42oQyzQSnHPnZ/vqoCdDmzH28fzWByN9asMeM8A= +google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= google.golang.org/genproto v0.0.0-20221118155620-16455021b5e6 h1:a2S6M0+660BgMNl++4JPlcAO/CjkqYItDEZwkoDQK7c= google.golang.org/genproto v0.0.0-20221118155620-16455021b5e6/go.mod h1:rZS5c/ZVYMaOGBfO68GWtjOw/eLaZM1X6iVtgjZ+EWg= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= @@ -1220,9 +1711,11 @@ google.golang.org/grpc v1.33.1/go.mod h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTp google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= google.golang.org/grpc v1.34.0/go.mod h1:WotjhfgOW/POjDeRt8vscBtXq+2VjORFy659qA51WJ8= google.golang.org/grpc v1.35.0-dev.0.20201218190559-666aea1fb34c/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= +google.golang.org/grpc v1.35.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= google.golang.org/grpc v1.36.1/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= google.golang.org/grpc v1.37.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= +google.golang.org/grpc v1.38.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= google.golang.org/grpc v1.52.3 h1:pf7sOysg4LdgBqduXveGKrcEwbStiK2rtfghdzlUYDQ= google.golang.org/grpc v1.52.3/go.mod h1:pu6fVzoFb+NBYNAvQL08ic+lvB2IojljRYuun5vorUY= google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.0.0/go.mod h1:6Kw0yEErY5E/yWrBtf03jp27GLLJujG4z/JK95pnjjw= @@ -1244,6 +1737,7 @@ google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQ google.golang.org/protobuf v1.28.1 h1:d0NfwRgPtno5B1Wa6L2DAG+KivqkdutMf1UhdNx175w= google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= +gopkg.in/bsm/ratelimit.v1 v1.0.0-20160220154919-db14e161995a/go.mod h1:KF9sEfUPAXdG8Oev9e99iLGnl2uJMjc5B+4y3O7x610= gopkg.in/cenkalti/backoff.v1 v1.1.0 h1:Arh75ttbsvlpVA7WtVpH4u9h6Zl46xuptxqLxPiSo4Y= gopkg.in/cenkalti/backoff.v1 v1.1.0/go.mod h1:J6Vskwqd+OMVJl8C33mmtxTBs2gyzfv7UDAkHu8BrjI= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= @@ -1251,6 +1745,7 @@ gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8 gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/go-playground/assert.v1 v1.2.1/go.mod h1:9RXL0bg/zibRAgZUYszZSwO/z8Y/a8bDuhia5mkpMnE= @@ -1259,19 +1754,29 @@ gopkg.in/inconshreveable/log15.v2 v2.0.0-20180818164646-67afb5ed74ec/go.mod h1:a gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/ini.v1 v1.61.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= +gopkg.in/ini.v1 v1.62.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= +gopkg.in/jcmturner/aescts.v1 v1.0.1/go.mod h1:nsR8qBOg+OucoIW+WMhB3GspUQXq9XorLnQb9XtvcOo= +gopkg.in/jcmturner/dnsutils.v1 v1.0.1/go.mod h1:m3v+5svpVOhtFAP/wSz+yzh4Mc0Fg7eRhxkJMWSIz9Q= +gopkg.in/jcmturner/goidentity.v3 v3.0.0/go.mod h1:oG2kH0IvSYNIu80dVAyu/yoefjq1mNfM5bm88whjWx4= +gopkg.in/jcmturner/gokrb5.v7 v7.5.0/go.mod h1:l8VISx+WGYp+Fp7KRbsiUuXTTOnxIc3Tuvyavf11/WM= +gopkg.in/jcmturner/rpc.v1 v1.1.0/go.mod h1:YIdkC4XfD6GXbzje11McwsDuOlZQSb9W4vfLvuNnlv8= gopkg.in/natefinch/npipe.v2 v2.0.0-20160621034901-c1b8fa8bdcce h1:+JknDZhAj8YMt7GC73Ei8pv4MzjDUNPHgQWJdtMAaDU= gopkg.in/natefinch/npipe.v2 v2.0.0-20160621034901-c1b8fa8bdcce/go.mod h1:5AcXVHNjg+BDxry382+8OKon8SEWiKktQR07RKPsv1c= +gopkg.in/olebedev/go-duktape.v3 v3.0.0-20200619000410-60c24ae608a6/go.mod h1:uAJfkITjFhyEEuUfm7bsmCZRbW5WRq8s9EY8HZ6hCns= +gopkg.in/redis.v4 v4.2.4/go.mod h1:8KREHdypkCEojGKQcjMqAODMICIVwZAONWq8RowTITA= gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= +gopkg.in/urfave/cli.v1 v1.20.0/go.mod h1:vuBzUtMdQeixQj8LVd+/98pzhxNGQoyuPBlsXHOQNO0= gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= @@ -1290,12 +1795,14 @@ honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWh honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= +honnef.co/go/tools v0.1.3/go.mod h1:NgwopIslSNH47DimFoV78dnkksY2EFtX0ajyb3K/las= k8s.io/apimachinery v0.25.0 h1:MlP0r6+3XbkUG2itd6vp3oxbtdQLQI94fD5gCS+gnoU= k8s.io/client-go v0.25.0 h1:CVWIaCETLMBNiTUta3d5nzRbXvY5Hy9Dpl+VvREpu5E= k8s.io/client-go v0.25.0/go.mod h1:lxykvypVfKilxhTklov0wz1FoaUZ8X4EwbhS6rpRfN8= k8s.io/klog v1.0.0 h1:Pt+yjF5aB1xDSVbau4VsWe+dQNzA0qv1LlXdC2dF6Q8= k8s.io/klog/v2 v2.70.1 h1:7aaoSdahviPmR+XkS7FyxlkkXs6tHISSG03RxleQAVQ= k8s.io/utils v0.0.0-20220728103510-ee6ede2d64ed h1:jAne/RjBTyawwAy0utX5eqigAwz/lQhTmy+Hr/Cpue4= +launchpad.net/gocheck v0.0.0-20140225173054-000000000087/go.mod h1:hj7XX3B/0A+80Vse0e+BUHsHMTEhd0O4cpUHr/e/BUM= lukechampine.com/blake3 v1.1.7 h1:GgRMhmdsuK8+ii6UZFDL8Nb+VyMwadAgcJyfYHxG6n0= lukechampine.com/blake3 v1.1.7/go.mod h1:tkKEOtDkNtklkXtLNEOGNq5tcV90tJiA1vAA12R78LA= lukechampine.com/uint128 v1.2.0 h1:mBi/5l91vocEN8otkC5bDLhi2KdCticRiwbdB0O+rjI= @@ -1310,6 +1817,7 @@ modernc.org/strutil v1.1.3 h1:fNMm+oJklMGYfU9Ylcywl0CO5O6nTfaowNsh2wpPjzY= modernc.org/token v1.1.0 h1:Xl7Ap9dKaEs5kLoOQeQmPWevfnk/DM5qcLcYlA8ys6Y= rsc.io/binaryregexp v0.2.0 h1:HfqmD5MEmC0zvwBuF187nq9mdnXjXsSivRiXN7SmRkE= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= +rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= sigs.k8s.io/json v0.0.0-20220713155537-f223a00ba0e2 h1:iXTIw73aPyC+oRdyqqvVJuloN1p0AC/kzH07hu3NE+k= diff --git a/local-deployment/README.md b/local-deployment/README.md index 1a7ca5257f..a672f27316 100644 --- a/local-deployment/README.md +++ b/local-deployment/README.md @@ -66,3 +66,11 @@ Exiting individual validators can be done using the provided `exit_validator.sh` ``` bash exit_validators.sh -i validator_index -m "memonic" -b "http://bn_api_host:bn_api_port" ``` + +# Enabling withdrawals +The script `add_withdrawal_address.sh` allows you to create & submit a bls to execution layer address change message in order to enable withdrawals for specific validators.. Requires [/github.com/protolambda/eth2-val-tools](eth2-val-tools) to be available on the path. +``` +go get github.com/protolambda/eth2-val-tools@master +go install github.com/protolambda/eth2-val-tools@master +bash add_withdrawal_address.sh -a "EL address (0x1234...)" -i validator_index -m "memonic" -b "http://bn_api_host:bn_api_port" +``` diff --git a/local-deployment/network-params.json b/local-deployment/network-params.json index 9900af36a0..0fc7779931 100644 --- a/local-deployment/network-params.json +++ b/local-deployment/network-params.json @@ -11,7 +11,7 @@ "el_extra_params": ["--http.api='admin,eth,net,debug,txpool' --syncmode full --gcmode archive"], "validator_extra_params": [], "builder_network_params": null, - "count": 1 + "count": 3 } ], "network_params": { @@ -21,7 +21,7 @@ "deposit_contract_address": "0x4242424242424242424242424242424242424242", "seconds_per_slot": 4, "genesis_delay": 10, - "capella_fork_epoch": 5, + "capella_fork_epoch": 3, "deneb_fork_epoch": 500 }, "global_client_log_level": "info", diff --git a/local-deployment/provision-explorer-config.sh b/local-deployment/provision-explorer-config.sh index 5ef12a2145..e51b508c61 100644 --- a/local-deployment/provision-explorer-config.sh +++ b/local-deployment/provision-explorer-config.sh @@ -1,17 +1,17 @@ #! /bin/bash -CL_PORT=$(kurtosis enclave inspect my-testnet | grep 4000/tcp | tr -s ' ' | cut -d " " -f 6 | sed -e 's/http\:\/\/127.0.0.1\://' | grep "\S") +CL_PORT=$(kurtosis enclave inspect my-testnet | grep 4000/tcp | tr -s ' ' | cut -d " " -f 6 | sed -e 's/http\:\/\/127.0.0.1\://' | head -n 1) echo "CL Node port is $CL_PORT" -EL_PORT=$(kurtosis enclave inspect my-testnet | grep 8545/tcp | tr -s ' ' | cut -d " " -f 5 | sed -e 's/127.0.0.1\://' | grep "\S") +EL_PORT=$(kurtosis enclave inspect my-testnet | grep 8545/tcp | tr -s ' ' | cut -d " " -f 5 | sed -e 's/127.0.0.1\://' | head -n 1) echo "EL Node port is $EL_PORT" -REDIS_PORT=$(kurtosis enclave inspect my-testnet | grep 6379/tcp | tr -s ' ' | cut -d " " -f 6 | sed -e 's/tcp\:\/\/127.0.0.1\://' | grep "\S") +REDIS_PORT=$(kurtosis enclave inspect my-testnet | grep 6379/tcp | tr -s ' ' | cut -d " " -f 6 | sed -e 's/tcp\:\/\/127.0.0.1\://' | head -n 1) echo "Redis port is $REDIS_PORT" -POSTGRES_PORT=$(kurtosis enclave inspect my-testnet | grep 5432/tcp | tr -s ' ' | cut -d " " -f 6 | sed -e 's/postgresql\:\/\/127.0.0.1\://' | grep "\S") +POSTGRES_PORT=$(kurtosis enclave inspect my-testnet | grep 5432/tcp | tr -s ' ' | cut -d " " -f 6 | sed -e 's/postgresql\:\/\/127.0.0.1\://' | head -n 1) echo "Postgres port is $POSTGRES_PORT" -LBT_PORT=$(kurtosis enclave inspect my-testnet | grep 9000/tcp | tr -s ' ' | cut -d " " -f 6 | sed -e 's/tcp\:\/\/127.0.0.1\://' | grep "\S") +LBT_PORT=$(kurtosis enclave inspect my-testnet | grep 9000/tcp | tr -s ' ' | cut -d " " -f 6 | sed -e 's/tcp\:\/\/127.0.0.1\://' | tail -n 1) echo "Little bigtable port is $LBT_PORT" touch config.yml diff --git a/local-deployment/scripts/add_withdrawal_addr.sh b/local-deployment/scripts/add_withdrawal_addr.sh new file mode 100644 index 0000000000..bb74779f36 --- /dev/null +++ b/local-deployment/scripts/add_withdrawal_addr.sh @@ -0,0 +1,37 @@ +#! /bin/bash +set -e + +clean_up () { + ARG=$? + rm -rf /tmp/set_withdrawal_addr + exit $ARG +} +trap clean_up EXIT + +while getopts a:b:i:m: flag +do + case "${flag}" in + a) el_address=${OPTARG};; + b) bn_endpoint=${OPTARG};; + i) index=${OPTARG};; + m) mnemonic=${OPTARG};; + esac +done +echo "Validator Index: $index"; +echo "Mnemonic: $mnemonic"; +echo "BN Endpoint: $bn_endpoint"; +echo "EL Withdrawal Address: $el_address"; + +mkdir -p /tmp/set_withdrawal_addr +echo "retrieving metadata" +genesis_validators_root=$(curl --silent $bn_endpoint/eth/v1/beacon/genesis | jq -r '.data.genesis_validators_root') +fork_version=$(curl --silent $bn_endpoint/eth/v1/beacon/genesis | jq -r '.data.genesis_fork_version') +deposit_contract_address=$(curl --silent $bn_endpoint/eth/v1/config/spec | jq -r '.data.DEPOSIT_CONTRACT_ADDRESS') +echo "generating bls-el change message" +eth2-val-tools bls-address-change --withdrawals-mnemonic="$mnemonic" --execution-address="$el_address" --source-min="$index" --source-max="$(($index + 1))" --genesis-validators-root="$genesis_validators_root" --fork-version="$fork_version" --as-json-list=true > "/tmp/set_withdrawal_addr/change_operations.json" +cat /tmp/set_withdrawal_addr/change_operations.json +echo "publishing bls-el change message" +curl -X POST $bn_endpoint/eth/v1/beacon/pool/bls_to_execution_changes \ + -H "Content-Type: application/json" \ + --data-binary "@/tmp/set_withdrawal_addr/change_operations.json" +rm -rf /tmp/set_withdrawal_addr \ No newline at end of file diff --git a/local-deployment/scripts/deposit_validator.sh b/local-deployment/scripts/deposit_validator.sh new file mode 100644 index 0000000000..f6f4ca4bc1 --- /dev/null +++ b/local-deployment/scripts/deposit_validator.sh @@ -0,0 +1,49 @@ +#! /bin/bash +set -e + +clean_up () { + ARG=$? + rm -rf /tmp/deposit + exit $ARG +} +trap clean_up EXIT + +while getopts a:b:e:i:m: flag +do + case "${flag}" in + b) bn_endpoint=${OPTARG};; + e) el_endpoint=${OPTARG};; + i) index=${OPTARG};; + m) mnemonic=${OPTARG};; + esac +done + +echo "Validator Index: $index"; +echo "Mnemonic: $mnemonic"; +echo "BN Endpoint: $bn_endpoint"; +echo "EL Endpoint: $el_endpoint"; + +mkdir -p /tmp/deposit +deposit_path="m/44'/60'/0'/0/3" +privatekey="ef5177cd0b6b21c87db5a0bf35d4084a8a57a9d6a064f86d51ac85f2b873a4e2" +publickey="0x878705ba3f8Bc32FCf7F4CAa1A35E72AF65CF766" +fork_version=$(curl -s $bn_endpoint/eth/v1/beacon/genesis | jq -r '.data.genesis_fork_version') +deposit_contract_address=$(curl -s $bn_endpoint/eth/v1/config/spec | jq -r '.data.DEPOSIT_CONTRACT_ADDRESS') +eth2-val-tools deposit-data --source-min=192 --source-max=200 --amount=32000000000 --fork-version=$fork_version --withdrawals-mnemonic="$mnemonic" --validators-mnemonic="$mnemonic" > /tmp/deposit/deposits_0-9.txt +while read x; do + account_name="$(echo "$x" | jq '.account')" + pubkey="$(echo "$x" | jq '.pubkey')" + echo "Sending deposit for validator $account_name $pubkey" + ethereal beacon deposit \ + --allow-unknown-contract=true \ + --address="$deposit_contract_address" \ + --connection=$el_endpoint \ + --data="$x" \ + --value="32000000000" \ + --from="$publickey" \ + --privatekey="$privatekey" + echo "Sent deposit for validator $account_name $pubkey" + sleep 3 +done < /tmp/deposit/deposits_0-9.txt +exit; +rm -rf /tmp/deposit \ No newline at end of file diff --git a/local-deployment/exit_validator.sh b/local-deployment/scripts/exit_validator.sh similarity index 100% rename from local-deployment/exit_validator.sh rename to local-deployment/scripts/exit_validator.sh diff --git a/templates/deposits.html b/templates/deposits.html index 2181df2704..75b1928a06 100644 --- a/templates/deposits.html +++ b/templates/deposits.html @@ -177,13 +177,13 @@

Top Depo

Runner Up

- {{ if len .Stats.TopDepositors }}{{ (index .Stats.TopDepositors 1).DepositCount }}{{ else }}0{{ end }} Deposits + {{ if gt (len .Stats.TopDepositors) 1 }}{{ (index .Stats.TopDepositors 1).DepositCount }}{{ else }}0{{ end }} Deposits
- {{ if len .Stats.TopDepositors }}0x{{ (index .Stats.TopDepositors 1).Address }}{{ end }} + {{ if gt (len .Stats.TopDepositors) 1 }}0x{{ (index .Stats.TopDepositors 1).Address }}{{ else }}N/A{{ end }}
From fd848ffa0e4dd5a298a10348b31db9af7eeda5f4 Mon Sep 17 00:00:00 2001 From: peter <1674920+peterbitfly@users.noreply.github.com> Date: Thu, 7 Sep 2023 09:03:15 +0200 Subject: [PATCH 079/106] (BIDS-2379) fix uint64 underflow during epoch 0 --- handlers/validator.go | 6 +++++- local-deployment/network-params.json | 2 +- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/handlers/validator.go b/handlers/validator.go index 4764d61956..1b8e53df39 100644 --- a/handlers/validator.go +++ b/handlers/validator.go @@ -1547,7 +1547,11 @@ func ValidatorHistory(w http.ResponseWriter, r *http.Request) { start = uint64((maxPages - 1) * pageLength) } - currentEpoch := services.LatestEpoch() - 1 + currentEpoch := services.LatestEpoch() + + if currentEpoch != 0 { + currentEpoch = currentEpoch - 1 + } var postExitEpochs uint64 = 0 // for an exited validator we show the history until his exit or (in rare cases) until his last sync / propose duties are finished if activationAndExitEpoch.ExitEpoch != 9223372036854775807 && currentEpoch > (activationAndExitEpoch.ExitEpoch-1) { diff --git a/local-deployment/network-params.json b/local-deployment/network-params.json index 0fc7779931..13dd9dbe58 100644 --- a/local-deployment/network-params.json +++ b/local-deployment/network-params.json @@ -11,7 +11,7 @@ "el_extra_params": ["--http.api='admin,eth,net,debug,txpool' --syncmode full --gcmode archive"], "validator_extra_params": [], "builder_network_params": null, - "count": 3 + "count": 1 } ], "network_params": { From 7bbda4bd6507297f1281b46b966c3fe083b130df Mon Sep 17 00:00:00 2001 From: peter <1674920+peterbitfly@users.noreply.github.com> Date: Sat, 9 Sep 2023 15:45:32 +0200 Subject: [PATCH 080/106] (BIDS-2379) rework bigtable schema to consolidate validator history data in one table, incorporate cr feedback, refactor handling of attestation & sync duties --- bigtable_config.md | 88 +----- cmd/explorer/main.go | 2 +- cmd/statistics/main.go | 3 +- db/bigtable.go | 504 +++++++++--------------------- db/bigtable_init.go | 28 +- db/db.go | 85 +---- db/statistics.go | 5 +- exporter/exporter.go | 63 ++-- exporter/sync_committees_count.go | 2 +- handlers/api.go | 8 +- handlers/api_eth1.go | 6 +- handlers/validator.go | 5 +- rpc/lighthouse.go | 202 +++++++++--- types/exporter.go | 51 +-- 14 files changed, 420 insertions(+), 632 deletions(-) diff --git a/bigtable_config.md b/bigtable_config.md index a3af5d0ace..f5f4e6702a 100644 --- a/bigtable_config.md +++ b/bigtable_config.md @@ -2,74 +2,31 @@ This document summarized the bigtable configuration options and table definitions required to run the beaconcha.in explorer. All settings can be applied either by using the GCP bigtable web interface or the `cbt` tool. ---- -Table name: `beaconchain_validator_balances` +Table name: `beaconchain_validator_history` ``` -cbt -project $PROJECT -instance $INSTANCE createtable beaconchain_validator_balances +cbt -project $PROJECT -instance $INSTANCE createtable beaconchain_validator_history ``` Column families: * Name: `vb` | GC Policy: None - -``` -cbt -project $PROJECT -instance $INSTANCE createfamily beaconchain_validator_balances vb -cbt -project $PROJECT -instance $INSTANCE createfamily beaconchain_validator_balances ha -``` - ----- -Table name: `beaconchain_validator_attestations` - -``` -cbt -project $PROJECT -instance $INSTANCE createtable beaconchain_validator_attestations -``` - -Column families: +* Name: `ha` | GC Policy: None * Name: `at` | GC Policy: None - -``` -cbt -project $PROJECT -instance $INSTANCE createfamily beaconchain_validator_attestations at -``` ----- -Table name: `beaconchain_validator_proposals` - -``` -cbt -project $PROJECT -instance $INSTANCE createtable beaconchain_validator_proposals -``` - -Column families: * Name: `pr` | GC Policy: None - -``` -cbt -project $PROJECT -instance $INSTANCE createfamily beaconchain_validator_proposals pr -``` ----- -Table name: `beaconchain_validator_sync` - -``` -cbt -project $PROJECT -instance $INSTANCE createtable beaconchain_validator_sync -``` - -Column families: * Name: `sc` | GC Policy: None - -``` -cbt -project $PROJECT -instance $INSTANCE createfamily beaconchain_validator_sync sc -cbt -project $PROJECT -instance $INSTANCE createfamily beaconchain_validator_sync sp -``` ----- -Table name: `beaconchain_validator_income` - -``` -cbt -project $PROJECT -instance $INSTANCE createtable beaconchain_validator_income -``` - -Column families: +* Name: `sp` | GC Policy: None * Name: `id` | GC Policy: None * Name: `stats` | GC Policy: None ``` -cbt -project $PROJECT -instance $INSTANCE createfamily beaconchain_validator_income id -cbt -project $PROJECT -instance $INSTANCE createfamily beaconchain_validator_income stats +cbt -project $PROJECT -instance $INSTANCE createfamily beaconchain_validator_history vb +cbt -project $PROJECT -instance $INSTANCE createfamily beaconchain_validator_history ha +cbt -project $PROJECT -instance $INSTANCE createfamily beaconchain_validator_history at +cbt -project $PROJECT -instance $INSTANCE createfamily beaconchain_validator_history pr +cbt -project $PROJECT -instance $INSTANCE createfamily beaconchain_validator_history sc +cbt -project $PROJECT -instance $INSTANCE createfamily beaconchain_validator_history sp +cbt -project $PROJECT -instance $INSTANCE createfamily beaconchain_validator_history id +cbt -project $PROJECT -instance $INSTANCE createfamily beaconchain_validator_history stats ``` ---- Table name: `beaconchain_validators` @@ -102,27 +59,6 @@ cbt -project $PROJECT -instance $INSTANCE createfamily blocks default cbt -project $PROJECT -instance $INSTANCE setgcpolicy blocks default maxversions=1 ``` ---- -Table name: `cache` - -``` -cbt -project $PROJECT -instance $INSTANCE createtable cache -``` - -Column families: -* Name: `10_min` | GC Policy: Version based policy with a maximum of 1 versions and a maximum age of 10 minutes -* Name: `1_day` | GC Policy: Version based policy with a maximum of 1 versions and a maximum age of 1 day -* Name: `1_hour` | GC Policy: Version based policy with a maximum of 1 versions and a maximum age of 1 hour - -``` -cbt -project $PROJECT -instance $INSTANCE createfamily cache 10_min -cbt -project $PROJECT -instance $INSTANCE createfamily cache 1_day -cbt -project $PROJECT -instance $INSTANCE createfamily cache 1_hour - -cbt -project $PROJECT -instance $INSTANCE setgcpolicy cache 10_min maxage=10m and maxversions=1 -cbt -project $PROJECT -instance $INSTANCE setgcpolicy cache 1_day maxage=1d and maxversions=1 -cbt -project $PROJECT -instance $INSTANCE setgcpolicy cache 1_hour maxage=1h and maxversions=1 -``` ----- Table name: `data` ``` diff --git a/cmd/explorer/main.go b/cmd/explorer/main.go index a0975be184..fbce5376fc 100644 --- a/cmd/explorer/main.go +++ b/cmd/explorer/main.go @@ -192,7 +192,7 @@ func main() { wg.Wait() if utils.Config.TieredCacheProvider != "redis" { - logrus.Fatalf("no cache provider set, please set TierdCacheProvider (example redis, bigtable)") + logrus.Fatalf("no cache provider set, please set TierdCacheProvider (example redis)") } defer db.ReaderDb.Close() diff --git a/cmd/statistics/main.go b/cmd/statistics/main.go index 8cd4f7ac09..7dd69d7937 100644 --- a/cmd/statistics/main.go +++ b/cmd/statistics/main.go @@ -237,8 +237,7 @@ func statisticsLoop(concurrencyTotal uint64, concurrencyCl uint64, concurrencyFa currentDay := latestEpoch / epochsPerDay previousDay := currentDay - 1 - logrus.Info(currentDay) - logrus.Info(previousDay) + logrus.Infof("Performing statisticsLoop with currentDay %v and previousDay %v", currentDay, previousDay) if previousDay > currentDay { previousDay = currentDay } diff --git a/db/bigtable.go b/db/bigtable.go index b7e903551b..e0291c1d2e 100644 --- a/db/bigtable.go +++ b/db/bigtable.go @@ -48,13 +48,9 @@ const ( type Bigtable struct { client *gcp_bigtable.Client - tableBeaconchain *gcp_bigtable.Table - tableValidators *gcp_bigtable.Table - tableValidatorBalances *gcp_bigtable.Table - tableValidatorAttestations *gcp_bigtable.Table - tableValidatorProposals *gcp_bigtable.Table - tableValidatorSyncCommittees *gcp_bigtable.Table - tableValidatorIncomeDetails *gcp_bigtable.Table + tableBeaconchain *gcp_bigtable.Table + tableValidators *gcp_bigtable.Table + tableValidatorsHistory *gcp_bigtable.Table tableData *gcp_bigtable.Table tableBlocks *gcp_bigtable.Table @@ -106,22 +102,18 @@ func InitBigtable(project, instance, chainId, redisAddress string) (*Bigtable, e } bt := &Bigtable{ - client: btClient, - tableData: btClient.Open("data"), - tableBlocks: btClient.Open("blocks"), - tableMetadataUpdates: btClient.Open("metadata_updates"), - tableMetadata: btClient.Open("metadata"), - tableBeaconchain: btClient.Open("beaconchain"), - tableMachineMetrics: btClient.Open("machine_metrics"), - tableValidators: btClient.Open("beaconchain_validators"), - tableValidatorBalances: btClient.Open("beaconchain_validator_balances"), - tableValidatorAttestations: btClient.Open("beaconchain_validator_attestations"), - tableValidatorProposals: btClient.Open("beaconchain_validator_proposals"), - tableValidatorSyncCommittees: btClient.Open("beaconchain_validator_sync"), - tableValidatorIncomeDetails: btClient.Open("beaconchain_validator_income"), - chainId: chainId, - redisCache: rdc, - lastAttestationCacheMux: &sync.Mutex{}, + client: btClient, + tableData: btClient.Open("data"), + tableBlocks: btClient.Open("blocks"), + tableMetadataUpdates: btClient.Open("metadata_updates"), + tableMetadata: btClient.Open("metadata"), + tableBeaconchain: btClient.Open("beaconchain"), + tableMachineMetrics: btClient.Open("machine_metrics"), + tableValidators: btClient.Open("beaconchain_validators"), + tableValidatorsHistory: btClient.Open("beaconchain_validators_history"), + chainId: chainId, + redisCache: rdc, + lastAttestationCacheMux: &sync.Mutex{}, } BigtableClient = bt @@ -443,13 +435,13 @@ func (bigtable *Bigtable) SaveValidatorBalances(epoch uint64, validators []*type combined := append(balanceEncoded, effectiveBalanceEncoded...) mut := &gcp_bigtable.Mutation{} mut.Set(VALIDATOR_BALANCES_FAMILY, "b", ts, combined) - key := fmt.Sprintf("%s:%s:%s", bigtable.chainId, bigtable.validatorIndexToKey(validator.Index), epochKey) + key := fmt.Sprintf("%s:%s:%s:%s", bigtable.chainId, bigtable.validatorIndexToKey(validator.Index), VALIDATOR_BALANCES_FAMILY, epochKey) muts = append(muts, mut) keys = append(keys, key) if len(muts) == MAX_BATCH_MUTATIONS { - errs, err := bigtable.tableValidatorBalances.ApplyBulk(ctx, keys, muts) + errs, err := bigtable.tableValidatorsHistory.ApplyBulk(ctx, keys, muts) if err != nil { return err @@ -463,14 +455,8 @@ func (bigtable *Bigtable) SaveValidatorBalances(epoch uint64, validators []*type } } - highestActiveIndexEncoded := make([]byte, 8) - binary.LittleEndian.PutUint64(highestActiveIndexEncoded, highestActiveIndex) - - mut := &gcp_bigtable.Mutation{} - mut.Set(VALIDATOR_HIGHEST_ACTIVE_INDEX_FAMILY, VALIDATOR_HIGHEST_ACTIVE_INDEX_FAMILY, ts, highestActiveIndexEncoded) - if len(muts) > 0 { - errs, err := bigtable.tableValidatorBalances.ApplyBulk(ctx, keys, muts) + errs, err := bigtable.tableValidatorsHistory.ApplyBulk(ctx, keys, muts) if err != nil { return err @@ -481,8 +467,14 @@ func (bigtable *Bigtable) SaveValidatorBalances(epoch uint64, validators []*type } } + // store the highes active validator index for that epoch + highestActiveIndexEncoded := make([]byte, 8) + binary.LittleEndian.PutUint64(highestActiveIndexEncoded, highestActiveIndex) + + mut := &gcp_bigtable.Mutation{} + mut.Set(VALIDATOR_HIGHEST_ACTIVE_INDEX_FAMILY, VALIDATOR_HIGHEST_ACTIVE_INDEX_FAMILY, ts, highestActiveIndexEncoded) key := fmt.Sprintf("%s:%s:%s", bigtable.chainId, VALIDATOR_HIGHEST_ACTIVE_INDEX_FAMILY, epochKey) - err := bigtable.tableValidatorBalances.Apply(ctx, key, mut) + err := bigtable.tableValidatorsHistory.Apply(ctx, key, mut) if err != nil { return err } @@ -491,73 +483,6 @@ func (bigtable *Bigtable) SaveValidatorBalances(epoch uint64, validators []*type return nil } -func (bigtable *Bigtable) SaveAttestationAssignments(epoch uint64, assignments map[string]uint64) error { - - ctx, cancel := context.WithTimeout(context.Background(), time.Minute*2) - defer cancel() - - start := time.Now() - ts := gcp_bigtable.Timestamp(0) - - validatorsPerSlot := make(map[uint64][]uint64) - for key, validator := range assignments { - keySplit := strings.Split(key, "-") - - attesterslot, err := strconv.ParseUint(keySplit[0], 10, 64) - if err != nil { - return err - } - - if validatorsPerSlot[attesterslot] == nil { - validatorsPerSlot[attesterslot] = make([]uint64, 0, len(assignments)/int(utils.Config.Chain.Config.SlotsPerEpoch)) - } - validatorsPerSlot[attesterslot] = append(validatorsPerSlot[attesterslot], validator) - } - - muts := make([]*gcp_bigtable.Mutation, 0, MAX_BATCH_MUTATIONS) - keys := make([]string, 0, MAX_BATCH_MUTATIONS) - - for slot, validators := range validatorsPerSlot { - - for _, validator := range validators { - mut := gcp_bigtable.NewMutation() - key := fmt.Sprintf("%s:%s:%s", bigtable.chainId, bigtable.validatorIndexToKey(validator), bigtable.reversedPaddedEpoch(epoch)) - mut.Set(ATTESTATIONS_FAMILY, fmt.Sprintf("%d", slot), ts, []byte{}) - - muts = append(muts, mut) - keys = append(keys, key) - } - - if len(muts) == MAX_BATCH_MUTATIONS { - errs, err := bigtable.tableValidatorAttestations.ApplyBulk(ctx, keys, muts) - - if err != nil { - return err - } - - for _, err := range errs { - return err - } - muts = make([]*gcp_bigtable.Mutation, 0, MAX_BATCH_MUTATIONS) - keys = make([]string, 0, MAX_BATCH_MUTATIONS) - } - } - - if len(muts) > 0 { - errs, err := bigtable.tableValidatorAttestations.ApplyBulk(ctx, keys, muts) - - if err != nil { - return err - } - for _, err := range errs { - return err - } - } - - logger.Infof("exported attestation assignments to bigtable in %v", time.Since(start)) - return nil -} - func (bigtable *Bigtable) SaveProposalAssignments(epoch uint64, assignments map[uint64]uint64) error { ctx, cancel := context.WithTimeout(context.Background(), time.Second*30) @@ -573,92 +498,43 @@ func (bigtable *Bigtable) SaveProposalAssignments(epoch uint64, assignments map[ mut := gcp_bigtable.NewMutation() mut.Set(PROPOSALS_FAMILY, "p", ts, []byte{}) - key := fmt.Sprintf("%s:%s:%s:%s", bigtable.chainId, bigtable.validatorIndexToKey(validator), bigtable.reversedPaddedEpoch(epoch), bigtable.reversedPaddedSlot(slot)) + key := fmt.Sprintf("%s:%s:%s:%s:%s", bigtable.chainId, bigtable.validatorIndexToKey(validator), PROPOSALS_FAMILY, bigtable.reversedPaddedEpoch(epoch), bigtable.reversedPaddedSlot(slot)) muts = append(muts, mut) keys = append(keys, key) - } - - errs, err := bigtable.tableValidatorProposals.ApplyBulk(ctx, keys, muts) - - if err != nil { - return err - } - - for _, err := range errs { - return err - } - - logger.Infof("exported proposal assignments to bigtable in %v", time.Since(start)) - return nil -} - -func (bigtable *Bigtable) SaveSyncCommitteesAssignments(startSlot, endSlot uint64, validators []uint64) error { - - ctx, cancel := context.WithTimeout(context.Background(), time.Minute*10) - defer cancel() - - start := time.Now() - ts := gcp_bigtable.Timestamp(0) - - muts := make([]*gcp_bigtable.Mutation, 0, MAX_BATCH_MUTATIONS) - keys := make([]string, 0, MAX_BATCH_MUTATIONS) - - totalMutations := (endSlot - startSlot) * uint64(len(validators)) - processedMutations := 0 - period := startSlot / utils.Config.Chain.Config.EpochsPerSyncCommitteePeriod / utils.Config.Chain.Config.SlotsPerEpoch - for i := startSlot; i <= endSlot; i++ { - for _, validator := range validators { - mut := gcp_bigtable.NewMutation() - mut.Set(SYNC_COMMITTEES_FAMILY, "s", ts, []byte{}) - - key := fmt.Sprintf("%s:%s:%s:%s", bigtable.chainId, bigtable.validatorIndexToKey(validator), bigtable.reversedPaddedEpoch(utils.EpochOfSlot(i)), bigtable.reversedPaddedSlot(i)) - - muts = append(muts, mut) - keys = append(keys, key) - processedMutations++ - - if len(muts) == MAX_BATCH_MUTATIONS { - logger.Infof("saving %v mutations for sync duties (%d / %d, period: %d)", len(muts), processedMutations, totalMutations, period) - errs, err := bigtable.tableValidatorSyncCommittees.ApplyBulk(ctx, keys, muts) - if err != nil { - return err - } + if len(muts) == MAX_BATCH_MUTATIONS { + errs, err := bigtable.tableValidatorsHistory.ApplyBulk(ctx, keys, muts) - for _, err := range errs { - if err != nil { - return err - } - } + if err != nil { + return err + } - muts = make([]*gcp_bigtable.Mutation, 0, MAX_BATCH_MUTATIONS) - keys = make([]string, 0, MAX_BATCH_MUTATIONS) + for _, err := range errs { + return err } + muts = make([]*gcp_bigtable.Mutation, 0, MAX_BATCH_MUTATIONS) + keys = make([]string, 0, MAX_BATCH_MUTATIONS) } - } if len(muts) > 0 { - logger.Infof("saving %v mutations for sync duties (%d / %d, period: %v)", len(muts), processedMutations, totalMutations, period) - errs, err := bigtable.tableValidatorSyncCommittees.ApplyBulk(ctx, keys, muts) + errs, err := bigtable.tableValidatorsHistory.ApplyBulk(ctx, keys, muts) if err != nil { return err } for _, err := range errs { - if err != nil { - return err - } + return err } } - logger.Infof("exported sync committee assignments to bigtable in %v", time.Since(start)) + logger.Infof("exported proposal assignments to bigtable in %v", time.Since(start)) return nil } -func (bigtable *Bigtable) SaveAttestations(blocks map[uint64]map[string]*types.Block) error { +func (bigtable *Bigtable) SaveAttestationDuties(duties map[types.Slot]map[types.ValidatorIndex][]types.Slot) error { // Initialize in memory last attestation cache lazily bigtable.lastAttestationCacheMux.Lock() @@ -681,95 +557,66 @@ func (bigtable *Bigtable) SaveAttestations(blocks map[uint64]map[string]*types.B start := time.Now() - attestationsBySlot := make(map[uint64]map[uint64]uint64) //map[attestedSlot]map[validator]includedSlot - - slots := make([]uint64, 0, len(blocks)) - for slot := range blocks { - slots = append(slots, slot) - } - sort.Slice(slots, func(i, j int) bool { - return slots[i] < slots[j] - }) - - for _, slot := range slots { - for _, b := range blocks[slot] { - if b.Status == 2 { - if attestationsBySlot[slot] == nil { - attestationsBySlot[slot] = make(map[uint64]uint64) - } - for _, validator := range b.EmptySlotAttestationAssignments { - attestationsBySlot[slot][validator] = MAX_BLOCK_NUMBER // will result in the ts of the set bigtable cell being 0 - } - } else { - // logger.Infof("processing slot %v", slot) - for _, a := range b.Attestations { - for _, validator := range a.Attesters { - inclusionSlot := slot - attestedSlot := a.Data.Slot - if attestationsBySlot[attestedSlot] == nil { - attestationsBySlot[attestedSlot] = make(map[uint64]uint64) - } - - if attestationsBySlot[attestedSlot][validator] == 0 || inclusionSlot < attestationsBySlot[attestedSlot][validator] { - attestationsBySlot[attestedSlot][validator] = inclusionSlot - } - } - } - } - } - } - mutsInclusionSlot := make([]*gcp_bigtable.Mutation, 0, MAX_BATCH_MUTATIONS) keysInclusionSlot := make([]string, 0, MAX_BATCH_MUTATIONS) - for attestedSlot, inclusions := range attestationsBySlot { - mutLastAttestationSlot := gcp_bigtable.NewMutation() - mutLastAttestationSlotSet := false + writes := 0 - epoch := utils.EpochOfSlot(attestedSlot) - bigtable.lastAttestationCacheMux.Lock() - for validator, inclusionSlot := range inclusions { + mutLastAttestationSlotSet := false + mutLastAttestationSlot := gcp_bigtable.NewMutation() - mutInclusionSlot := gcp_bigtable.NewMutation() - mutInclusionSlot.Set(ATTESTATIONS_FAMILY, fmt.Sprintf("%d", attestedSlot), gcp_bigtable.Timestamp((MAX_BLOCK_NUMBER-inclusionSlot)*1000), []byte{}) - key := fmt.Sprintf("%s:%s:%s", bigtable.chainId, bigtable.validatorIndexToKey(validator), bigtable.reversedPaddedEpoch(epoch)) + for attestedSlot, validators := range duties { + for validator, inclusions := range validators { - mutsInclusionSlot = append(mutsInclusionSlot, mutInclusionSlot) - keysInclusionSlot = append(keysInclusionSlot, key) - if attestedSlot > bigtable.lastAttestationCache[validator] { - mutLastAttestationSlot.Set(ATTESTATIONS_FAMILY, fmt.Sprintf("%d", validator), gcp_bigtable.Timestamp((attestedSlot)*1000), []byte{}) - bigtable.lastAttestationCache[validator] = attestedSlot - mutLastAttestationSlotSet = true + epoch := utils.EpochOfSlot(uint64(attestedSlot)) + bigtable.lastAttestationCacheMux.Lock() + if len(inclusions) == 0 { // for missed attestations we write the max block number which will yield a cell ts of 0 + inclusions = append(inclusions, MAX_BLOCK_NUMBER) } + for _, inclusionSlot := range inclusions { + mutInclusionSlot := gcp_bigtable.NewMutation() + mutInclusionSlot.Set(ATTESTATIONS_FAMILY, fmt.Sprintf("%d", attestedSlot), gcp_bigtable.Timestamp((MAX_BLOCK_NUMBER-inclusionSlot)*1000), []byte{}) + key := fmt.Sprintf("%s:%s:%s:%s", bigtable.chainId, bigtable.validatorIndexToKey(uint64(validator)), ATTESTATIONS_FAMILY, bigtable.reversedPaddedEpoch(epoch)) + + mutsInclusionSlot = append(mutsInclusionSlot, mutInclusionSlot) + keysInclusionSlot = append(keysInclusionSlot, key) + writes++ + + if uint64(attestedSlot) > bigtable.lastAttestationCache[uint64(validator)] { + mutLastAttestationSlot.Set(ATTESTATIONS_FAMILY, fmt.Sprintf("%d", validator), gcp_bigtable.Timestamp((attestedSlot)*1000), []byte{}) + bigtable.lastAttestationCache[uint64(validator)] = uint64(attestedSlot) + mutLastAttestationSlotSet = true + } - } - bigtable.lastAttestationCacheMux.Unlock() - - attstart := time.Now() + if len(mutsInclusionSlot) == MAX_BATCH_MUTATIONS { + attstart := time.Now() + errs, err := bigtable.tableValidatorsHistory.ApplyBulk(ctx, keysInclusionSlot, mutsInclusionSlot) + if err != nil { + return err + } + for _, err := range errs { + return err + } + logger.Infof("applied attestation mutations in %v", time.Since(attstart)) + mutsInclusionSlot = make([]*gcp_bigtable.Mutation, 0, MAX_BATCH_MUTATIONS) + keysInclusionSlot = make([]string, 0, MAX_BATCH_MUTATIONS) + } - if len(mutsInclusionSlot) == MAX_BATCH_MUTATIONS { - errs, err := bigtable.tableValidatorAttestations.ApplyBulk(ctx, keysInclusionSlot, mutsInclusionSlot) - if err != nil { - return err } - for _, err := range errs { - return err - } - logger.Infof("applied attestation mutations in %v", time.Since(attstart)) - mutsInclusionSlot = make([]*gcp_bigtable.Mutation, 0, MAX_BATCH_MUTATIONS) - keysInclusionSlot = make([]string, 0, MAX_BATCH_MUTATIONS) + bigtable.lastAttestationCacheMux.Unlock() } - if mutLastAttestationSlotSet { - err := bigtable.tableValidators.Apply(ctx, fmt.Sprintf("%s:lastAttestationSlot", bigtable.chainId), mutLastAttestationSlot) - if err != nil { - return err - } + } + + if mutLastAttestationSlotSet { + err := bigtable.tableValidators.Apply(ctx, fmt.Sprintf("%s:lastAttestationSlot", bigtable.chainId), mutLastAttestationSlot) + if err != nil { + return err } } if len(mutsInclusionSlot) > 0 { attstart := time.Now() - errs, err := bigtable.tableValidatorAttestations.ApplyBulk(ctx, keysInclusionSlot, mutsInclusionSlot) + errs, err := bigtable.tableValidatorsHistory.ApplyBulk(ctx, keysInclusionSlot, mutsInclusionSlot) if err != nil { return err } @@ -779,7 +626,7 @@ func (bigtable *Bigtable) SaveAttestations(blocks map[uint64]map[string]*types.B logger.Infof("applied attestation mutations in %v", time.Since(attstart)) } - logger.Infof("exported attestations (new) to bigtable in %v", time.Since(start)) + logger.Infof("exported %v attestations to bigtable in %v", writes, time.Since(start)) return nil } @@ -824,69 +671,50 @@ func (bigtable *Bigtable) SaveProposals(blocks map[uint64]map[string]*types.Bloc } mut := gcp_bigtable.NewMutation() mut.Set(PROPOSALS_FAMILY, "b", gcp_bigtable.Timestamp((MAX_BLOCK_NUMBER-b.Slot)*1000), []byte{}) - key := fmt.Sprintf("%s:%s:%s:%s", bigtable.chainId, bigtable.validatorIndexToKey(b.Proposer), bigtable.reversedPaddedEpoch(utils.EpochOfSlot(b.Slot)), bigtable.reversedPaddedSlot(b.Slot)) + key := fmt.Sprintf("%s:%s:%s:%s:%s", bigtable.chainId, bigtable.validatorIndexToKey(b.Proposer), PROPOSALS_FAMILY, bigtable.reversedPaddedEpoch(utils.EpochOfSlot(b.Slot)), bigtable.reversedPaddedSlot(b.Slot)) muts = append(muts, mut) keys = append(keys, key) + + if len(muts) == MAX_BATCH_MUTATIONS { + errs, err := bigtable.tableValidatorsHistory.ApplyBulk(ctx, keys, muts) + + if err != nil { + return err + } + + for _, err := range errs { + return err + } + muts = make([]*gcp_bigtable.Mutation, 0, MAX_BATCH_MUTATIONS) + keys = make([]string, 0, MAX_BATCH_MUTATIONS) + } } } - errs, err := bigtable.tableValidatorProposals.ApplyBulk(ctx, keys, muts) - if err != nil { - return err - } + if len(muts) > 0 { + errs, err := bigtable.tableValidatorsHistory.ApplyBulk(ctx, keys, muts) - for _, err := range errs { - return err + if err != nil { + return err + } + + for _, err := range errs { + return err + } } logger.Infof("exported proposals to bigtable in %v", time.Since(start)) return nil } -func (bigtable *Bigtable) SaveSyncComitteeDuties(blocks map[uint64]map[string]*types.Block) error { +func (bigtable *Bigtable) SaveSyncComitteeDuties(duties map[types.Slot]map[types.ValidatorIndex]bool) error { ctx, cancel := context.WithTimeout(context.Background(), time.Minute*2) defer cancel() start := time.Now() - dutiesBySlot := make(map[uint64]map[uint64]bool) //map[dutiesSlot]map[validator]bool - - slots := make([]uint64, 0, len(blocks)) - for slot := range blocks { - slots = append(slots, slot) - } - sort.Slice(slots, func(i, j int) bool { - return slots[i] < slots[j] - }) - - for _, slot := range slots { - for _, b := range blocks[slot] { - if b.Status == 2 { - - if dutiesBySlot[b.Slot] == nil { - dutiesBySlot[b.Slot] = make(map[uint64]bool) - } - for _, validator := range b.EmptySlotSyncAssignments { - dutiesBySlot[b.Slot][validator] = false - } - } else if b.SyncAggregate != nil && len(b.SyncAggregate.SyncCommitteeValidators) > 0 { - bitLen := len(b.SyncAggregate.SyncCommitteeBits) * 8 - valLen := len(b.SyncAggregate.SyncCommitteeValidators) - if bitLen < valLen { - return fmt.Errorf("error getting sync_committee participants: bitLen != valLen: %v != %v", bitLen, valLen) - } - for i, valIndex := range b.SyncAggregate.SyncCommitteeValidators { - if dutiesBySlot[b.Slot] == nil { - dutiesBySlot[b.Slot] = make(map[uint64]bool) - } - dutiesBySlot[b.Slot][valIndex] = utils.BitAtVector(b.SyncAggregate.SyncCommitteeBits, i) - } - } - } - } - - if len(dutiesBySlot) == 0 { + if len(duties) == 0 { logger.Infof("no sync duties to export") return nil } @@ -894,7 +722,7 @@ func (bigtable *Bigtable) SaveSyncComitteeDuties(blocks map[uint64]map[string]*t muts := make([]*gcp_bigtable.Mutation, 0, utils.Config.Chain.Config.SlotsPerEpoch*utils.Config.Chain.Config.SyncCommitteeSize+1) keys := make([]string, 0, utils.Config.Chain.Config.SlotsPerEpoch*utils.Config.Chain.Config.SyncCommitteeSize+1) - for slot, validators := range dutiesBySlot { + for slot, validators := range duties { participation := uint64(0) for validator, participated := range validators { mut := gcp_bigtable.NewMutation() @@ -904,13 +732,13 @@ func (bigtable *Bigtable) SaveSyncComitteeDuties(blocks map[uint64]map[string]*t } else { mut.Set(SYNC_COMMITTEES_FAMILY, "s", gcp_bigtable.Timestamp(0), []byte{}) } - key := fmt.Sprintf("%s:%s:%s:%s", bigtable.chainId, bigtable.validatorIndexToKey(validator), bigtable.reversedPaddedEpoch(utils.EpochOfSlot(slot)), bigtable.reversedPaddedSlot(slot)) + key := fmt.Sprintf("%s:%s:%s:%s:%s", bigtable.chainId, bigtable.validatorIndexToKey(uint64(validator)), SYNC_COMMITTEES_FAMILY, bigtable.reversedPaddedEpoch(utils.EpochOfSlot(uint64(slot))), bigtable.reversedPaddedSlot(uint64(slot))) muts = append(muts, mut) keys = append(keys, key) } mut := gcp_bigtable.NewMutation() - key := fmt.Sprintf("%s:%s:%s", bigtable.chainId, SYNC_COMMITTEES_PARTICIPATION_FAMILY, bigtable.reversedPaddedSlot(slot)) + key := fmt.Sprintf("%s:%s:%s", bigtable.chainId, SYNC_COMMITTEES_PARTICIPATION_FAMILY, bigtable.reversedPaddedSlot(uint64(slot))) participationEncoded := make([]byte, 8) binary.LittleEndian.PutUint64(participationEncoded, uint64(participation)) mut.Set(SYNC_COMMITTEES_PARTICIPATION_FAMILY, "s", gcp_bigtable.Timestamp(0), participationEncoded) @@ -918,7 +746,7 @@ func (bigtable *Bigtable) SaveSyncComitteeDuties(blocks map[uint64]map[string]*t keys = append(keys, key) } - errs, err := bigtable.tableValidatorSyncCommittees.ApplyBulk(ctx, keys, muts) + errs, err := bigtable.tableValidatorsHistory.ApplyBulk(ctx, keys, muts) if err != nil { return err @@ -940,7 +768,7 @@ func (bigtable *Bigtable) GetMaxValidatorindexForEpoch(epoch uint64) (uint64, er key := fmt.Sprintf("%s:%s:%s", bigtable.chainId, VALIDATOR_HIGHEST_ACTIVE_INDEX_FAMILY, bigtable.reversedPaddedEpoch(epoch)) - row, err := bigtable.tableValidatorBalances.ReadRow(ctx, key) + row, err := bigtable.tableValidatorsHistory.ReadRow(ctx, key) if err != nil { return 0, err } @@ -984,13 +812,13 @@ func (bigtable *Bigtable) GetValidatorBalanceHistory(validators []uint64, startE return gCtx.Err() default: } - ranges := bigtable.getValidatorsEpochRanges(vals, startEpoch, endEpoch) + ranges := bigtable.getValidatorsEpochRanges(vals, VALIDATOR_BALANCES_FAMILY, startEpoch, endEpoch) ro := gcp_bigtable.LimitRows(int64(endEpoch-startEpoch+1) * int64(len(vals))) handleRow := func(r gcp_bigtable.Row) bool { keySplit := strings.Split(r.Key(), ":") - epoch, err := strconv.ParseUint(keySplit[2], 10, 64) + epoch, err := strconv.ParseUint(keySplit[3], 10, 64) if err != nil { logger.Errorf("error parsing epoch from row key %v: %v", r.Key(), err) return false @@ -1029,7 +857,7 @@ func (bigtable *Bigtable) GetValidatorBalanceHistory(validators []uint64, startE return true } - err := bigtable.tableValidatorBalances.ReadRows(gCtx, ranges, handleRow, ro) + err := bigtable.tableValidatorsHistory.ReadRows(gCtx, ranges, handleRow, ro) if err != nil { return err } @@ -1090,8 +918,8 @@ func (bigtable *Bigtable) GetValidatorAttestationHistory(validators []uint64, st return gCtx.Err() default: } - ranges := bigtable.getValidatorsEpochRanges(vals, startEpoch, endEpoch) - err = bigtable.tableValidatorAttestations.ReadRows(ctx, ranges, func(r gcp_bigtable.Row) bool { + ranges := bigtable.getValidatorsEpochRanges(vals, ATTESTATIONS_FAMILY, startEpoch, endEpoch) + err = bigtable.tableValidatorsHistory.ReadRows(ctx, ranges, func(r gcp_bigtable.Row) bool { keySplit := strings.Split(r.Key(), ":") validator, err := bigtable.validatorKeyToIndex(keySplit[1]) @@ -1213,25 +1041,6 @@ func (bigtable *Bigtable) GetLastAttestationSlots(validators []uint64) (map[uint return res, nil } -func (bigtable *Bigtable) GetSyncParticipationBySlot(slot uint64) (uint64, error) { - - ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(time.Minute*5)) - defer cancel() - - key := fmt.Sprintf("%s:%s:%s", bigtable.chainId, SYNC_COMMITTEES_PARTICIPATION_FAMILY, bigtable.reversedPaddedSlot(slot)) - - row, err := bigtable.tableValidatorSyncCommittees.ReadRow(ctx, key) - if err != nil { - return 0, err - } - - for _, ri := range row[SYNC_COMMITTEES_PARTICIPATION_FAMILY] { - return binary.LittleEndian.Uint64(ri.Value), nil - } - - return 0, nil -} - func (bigtable *Bigtable) GetSyncParticipationBySlotRange(startSlot, endSlot uint64) (map[uint64]uint64, error) { ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(time.Minute*5)) @@ -1242,7 +1051,7 @@ func (bigtable *Bigtable) GetSyncParticipationBySlotRange(startSlot, endSlot uin rowRange := gcp_bigtable.NewRange(startKey, endKey) res := make(map[uint64]uint64) - err := bigtable.tableValidatorSyncCommittees.ReadRows(ctx, rowRange, func(r gcp_bigtable.Row) bool { + err := bigtable.tableValidatorsHistory.ReadRows(ctx, rowRange, func(r gcp_bigtable.Row) bool { keySplit := strings.Split(r.Key(), ":") slot, err := strconv.ParseUint(keySplit[2], 10, 64) @@ -1311,8 +1120,8 @@ func (bigtable *Bigtable) GetValidatorMissedAttestationHistory(validators []uint return gCtx.Err() default: } - ranges := bigtable.getValidatorsEpochRanges(vals, startEpoch, endEpoch) - err = bigtable.tableValidatorAttestations.ReadRows(ctx, ranges, func(r gcp_bigtable.Row) bool { + ranges := bigtable.getValidatorsEpochRanges(vals, ATTESTATIONS_FAMILY, startEpoch, endEpoch) + err = bigtable.tableValidatorsHistory.ReadRows(ctx, ranges, func(r gcp_bigtable.Row) bool { keySplit := strings.Split(r.Key(), ":") validator, err := bigtable.validatorKeyToIndex(keySplit[1]) @@ -1402,9 +1211,9 @@ func (bigtable *Bigtable) GetValidatorSyncDutiesHistory(validators []uint64, sta return gCtx.Err() default: } - ranges := bigtable.getValidatorSlotRanges(vals, startSlot, endSlot) + ranges := bigtable.getValidatorSlotRanges(vals, SYNC_COMMITTEES_FAMILY, startSlot, endSlot) - err := bigtable.tableValidatorSyncCommittees.ReadRows(ctx, ranges, func(r gcp_bigtable.Row) bool { + err := bigtable.tableValidatorsHistory.ReadRows(ctx, ranges, func(r gcp_bigtable.Row) bool { for _, ri := range r[SYNC_COMMITTEES_FAMILY] { keySplit := strings.Split(r.Key(), ":") @@ -1688,8 +1497,8 @@ func (bigtable *Bigtable) GetValidatorProposalHistory(validators []uint64, start return gCtx.Err() default: } - ranges := bigtable.getValidatorsEpochSlotRanges(vals, startEpoch, endEpoch) - err := bigtable.tableValidatorProposals.ReadRows(ctx, ranges, func(r gcp_bigtable.Row) bool { + ranges := bigtable.getValidatorsEpochSlotRanges(vals, PROPOSALS_FAMILY, startEpoch, endEpoch) + err := bigtable.tableValidatorsHistory.ReadRows(ctx, ranges, func(r gcp_bigtable.Row) bool { for _, ri := range r[PROPOSALS_FAMILY] { keySplit := strings.Split(r.Key(), ":") @@ -1757,7 +1566,6 @@ func (bigtable *Bigtable) SaveValidatorIncomeDetails(epoch uint64, rewards map[u muts := make([]*gcp_bigtable.Mutation, 0, MAX_BATCH_MUTATIONS) keys := make([]string, 0, MAX_BATCH_MUTATIONS) - mutsCount := 0 for i, rewardDetails := range rewards { data, err := proto.Marshal(rewardDetails) @@ -1768,14 +1576,13 @@ func (bigtable *Bigtable) SaveValidatorIncomeDetails(epoch uint64, rewards map[u mut := &gcp_bigtable.Mutation{} mut.Set(INCOME_DETAILS_COLUMN_FAMILY, "i", ts, data) - key := fmt.Sprintf("%s:%s:%s", bigtable.chainId, bigtable.validatorIndexToKey(i), bigtable.reversedPaddedEpoch(epoch)) + key := fmt.Sprintf("%s:%s:%s:%s", bigtable.chainId, bigtable.validatorIndexToKey(i), INCOME_DETAILS_COLUMN_FAMILY, bigtable.reversedPaddedEpoch(epoch)) muts = append(muts, mut) keys = append(keys, key) - mutsCount++ if len(muts) == MAX_BATCH_MUTATIONS { - errs, err := bigtable.tableValidatorIncomeDetails.ApplyBulk(ctx, keys, muts) + errs, err := bigtable.tableValidatorsHistory.ApplyBulk(ctx, keys, muts) if err != nil { return err @@ -1804,7 +1611,7 @@ func (bigtable *Bigtable) SaveValidatorIncomeDetails(epoch uint64, rewards map[u } if len(muts) > 0 { - errs, err := bigtable.tableValidatorIncomeDetails.ApplyBulk(ctx, keys, muts) + errs, err := bigtable.tableValidatorsHistory.ApplyBulk(ctx, keys, muts) if err != nil { return err @@ -1822,7 +1629,7 @@ func (bigtable *Bigtable) SaveValidatorIncomeDetails(epoch uint64, rewards map[u mut := &gcp_bigtable.Mutation{} mut.Set(STATS_COLUMN_FAMILY, SUM_COLUMN, ts, sum) - err = bigtable.tableValidatorIncomeDetails.Apply(ctx, fmt.Sprintf("%s:%s:%s", bigtable.chainId, SUM_COLUMN, bigtable.reversedPaddedEpoch(epoch)), mut) + err = bigtable.tableValidatorsHistory.Apply(ctx, fmt.Sprintf("%s:%s:%s", bigtable.chainId, SUM_COLUMN, bigtable.reversedPaddedEpoch(epoch)), mut) if err != nil { return err } @@ -1862,7 +1669,7 @@ func (bigtable *Bigtable) MigrateEpochSchemaV1ToV2(epoch uint64) error { keySplit := strings.Split(r.Key(), ":") - rowKeyEpoch, err := strconv.ParseUint(keySplit[3], 10, 64) + rowKeyEpoch, err := strconv.ParseUint(keySplit[4], 10, 64) if err != nil { logger.Errorf("error parsing epoch from row key %v: %v", r.Key(), err) return false @@ -1940,7 +1747,7 @@ func (bigtable *Bigtable) MigrateEpochSchemaV1ToV2(epoch uint64) error { keySplit := strings.Split(r.Key(), ":") - rowKeyEpoch, err := strconv.ParseUint(keySplit[2], 10, 64) + rowKeyEpoch, err := strconv.ParseUint(keySplit[3], 10, 64) if err != nil { logger.Errorf("error parsing epoch from row key %v: %v", r.Key(), err) return false @@ -2032,20 +1839,19 @@ func (bigtable *Bigtable) MigrateEpochSchemaV1ToV2(epoch uint64) error { // logger.Infof("migrated balance data in %v", time.Since(start)) // start = time.Now() - i := 0 mutsInclusionSlot := make([]*gcp_bigtable.Mutation, 0, MAX_BATCH_MUTATIONS) keysInclusionSlot := make([]string, 0, MAX_BATCH_MUTATIONS) for _, validator := range epochData { mutInclusionSlot := gcp_bigtable.NewMutation() mutInclusionSlot.Set(ATTESTATIONS_FAMILY, fmt.Sprintf("%d", validator.AttestationTargetSlot), gcp_bigtable.Timestamp((MAX_BLOCK_NUMBER-validator.AttestationInclusionSlot)*1000), []byte{}) - key := fmt.Sprintf("%s:%s:%s", bigtable.chainId, bigtable.validatorIndexToKey(validator.ValidatorIndex), bigtable.reversedPaddedEpoch(epoch)) + key := fmt.Sprintf("%s:%s:%s:%s", bigtable.chainId, ATTESTATIONS_FAMILY, bigtable.validatorIndexToKey(validator.ValidatorIndex), bigtable.reversedPaddedEpoch(epoch)) mutsInclusionSlot = append(mutsInclusionSlot, mutInclusionSlot) keysInclusionSlot = append(keysInclusionSlot, key) if len(mutsInclusionSlot) == MAX_BATCH_MUTATIONS { - errs, err := bigtable.tableValidatorAttestations.ApplyBulk(ctx, keysInclusionSlot, mutsInclusionSlot) + errs, err := bigtable.tableValidatorsHistory.ApplyBulk(ctx, keysInclusionSlot, mutsInclusionSlot) if err != nil { return err } @@ -2059,7 +1865,7 @@ func (bigtable *Bigtable) MigrateEpochSchemaV1ToV2(epoch uint64) error { } if len(mutsInclusionSlot) > 0 { - errs, err := bigtable.tableValidatorAttestations.ApplyBulk(ctx, keysInclusionSlot, mutsInclusionSlot) + errs, err := bigtable.tableValidatorsHistory.ApplyBulk(ctx, keysInclusionSlot, mutsInclusionSlot) if err != nil { return err @@ -2082,13 +1888,13 @@ func (bigtable *Bigtable) MigrateEpochSchemaV1ToV2(epoch uint64) error { for slot, inclusionSlot := range validator.Proposals { mut := gcp_bigtable.NewMutation() mut.Set(PROPOSALS_FAMILY, "b", gcp_bigtable.Timestamp((MAX_BLOCK_NUMBER-inclusionSlot)*1000), []byte{}) - key := fmt.Sprintf("%s:%s:%s:%s", bigtable.chainId, bigtable.validatorIndexToKey(validator.ValidatorIndex), bigtable.reversedPaddedEpoch(epoch), bigtable.reversedPaddedSlot(slot)) + key := fmt.Sprintf("%s:%s:%s:%s:%s", bigtable.chainId, bigtable.validatorIndexToKey(validator.ValidatorIndex), PROPOSALS_FAMILY, bigtable.reversedPaddedEpoch(epoch), bigtable.reversedPaddedSlot(slot)) mutsProposals = append(mutsProposals, mut) keysProposals = append(keysProposals, key) } } - errs, err := bigtable.tableValidatorProposals.ApplyBulk(ctx, keysProposals, mutsProposals) + errs, err := bigtable.tableValidatorsHistory.ApplyBulk(ctx, keysProposals, mutsProposals) if err != nil { return err @@ -2112,13 +1918,13 @@ func (bigtable *Bigtable) MigrateEpochSchemaV1ToV2(epoch uint64) error { mut := gcp_bigtable.NewMutation() mut.Set(SYNC_COMMITTEES_FAMILY, "s", gcp_bigtable.Timestamp((MAX_BLOCK_NUMBER-inclusionSlot)*1000), []byte{}) - key := fmt.Sprintf("%s:%s:%s:%s", bigtable.chainId, bigtable.validatorIndexToKey(validator.ValidatorIndex), bigtable.reversedPaddedEpoch(epoch), bigtable.reversedPaddedSlot(slot)) + key := fmt.Sprintf("%s:%s:%s:%s:%s", bigtable.chainId, bigtable.validatorIndexToKey(validator.ValidatorIndex), SYNC_COMMITTEES_FAMILY, bigtable.reversedPaddedEpoch(epoch), bigtable.reversedPaddedSlot(slot)) mutsSync = append(mutsSync, mut) keysSync = append(keysSync, key) } } - errs, err = bigtable.tableValidatorSyncCommittees.ApplyBulk(ctx, keysSync, mutsSync) + errs, err = bigtable.tableValidatorsHistory.ApplyBulk(ctx, keysSync, mutsSync) if err != nil { return err @@ -2190,8 +1996,8 @@ func (bigtable *Bigtable) GetValidatorIncomeDetailsHistory(validators []uint64, return gCtx.Err() default: } - ranges := bigtable.getValidatorsEpochRanges(vals, startEpoch, endEpoch) - err := bigtable.tableValidatorIncomeDetails.ReadRows(ctx, ranges, func(r gcp_bigtable.Row) bool { + ranges := bigtable.getValidatorsEpochRanges(vals, INCOME_DETAILS_COLUMN_FAMILY, startEpoch, endEpoch) + err := bigtable.tableValidatorsHistory.ReadRows(ctx, ranges, func(r gcp_bigtable.Row) bool { keySplit := strings.Split(r.Key(), ":") validator, err := bigtable.validatorKeyToIndex(keySplit[1]) @@ -2200,7 +2006,7 @@ func (bigtable *Bigtable) GetValidatorIncomeDetailsHistory(validators []uint64, return false } - epoch, err := strconv.ParseUint(keySplit[2], 10, 64) + epoch, err := strconv.ParseUint(keySplit[3], 10, 64) if err != nil { logger.Errorf("error parsing epoch from row key %v: %v", r.Key(), err) return false @@ -2302,46 +2108,46 @@ func (bigtable *Bigtable) DeleteEpoch(epoch uint64) error { return fmt.Errorf("NOT IMPLEMENTED") } -func (bigtable *Bigtable) getValidatorsEpochRanges(validatorIndices []uint64, startEpoch uint64, endEpoch uint64) gcp_bigtable.RowRangeList { +func (bigtable *Bigtable) getValidatorsEpochRanges(validatorIndices []uint64, prefix string, startEpoch uint64, endEpoch uint64) gcp_bigtable.RowRangeList { if endEpoch < startEpoch { // handle overflows startEpoch = 0 } - ranges := make(gcp_bigtable.RowRangeList, 0, int((endEpoch-startEpoch))*len(validatorIndices)) + ranges := make(gcp_bigtable.RowRangeList, 0, int((endEpoch-startEpoch))*len(validatorIndices)+1) for _, validatorIndex := range validatorIndices { validatorKey := bigtable.validatorIndexToKey(validatorIndex) // epochs are sorted descending, so start with the largest epoch and end with the smallest // add \x00 to make the range inclusive - rangeEnd := fmt.Sprintf("%s:%s:%s%s", bigtable.chainId, validatorKey, bigtable.reversedPaddedEpoch(startEpoch), "\x00") - rangeStart := fmt.Sprintf("%s:%s:%s", bigtable.chainId, validatorKey, bigtable.reversedPaddedEpoch(endEpoch)) + rangeEnd := fmt.Sprintf("%s:%s:%s:%s%s", bigtable.chainId, validatorKey, prefix, bigtable.reversedPaddedEpoch(startEpoch), "\x00") + rangeStart := fmt.Sprintf("%s:%s:%s:%s", bigtable.chainId, validatorKey, prefix, bigtable.reversedPaddedEpoch(endEpoch)) ranges = append(ranges, gcp_bigtable.NewRange(rangeStart, rangeEnd)) } return ranges } -func (bigtable *Bigtable) getValidatorsEpochSlotRanges(validatorIndices []uint64, startEpoch uint64, endEpoch uint64) gcp_bigtable.RowRangeList { +func (bigtable *Bigtable) getValidatorsEpochSlotRanges(validatorIndices []uint64, prefix string, startEpoch uint64, endEpoch uint64) gcp_bigtable.RowRangeList { if endEpoch < startEpoch { // handle overflows startEpoch = 0 } - ranges := make(gcp_bigtable.RowRangeList, 0, int((endEpoch-startEpoch))*len(validatorIndices)) + ranges := make(gcp_bigtable.RowRangeList, 0, int((endEpoch-startEpoch))*len(validatorIndices)+1) for _, validatorIndex := range validatorIndices { validatorKey := bigtable.validatorIndexToKey(validatorIndex) - rangeEnd := fmt.Sprintf("%s:%s:%s:%s%s", bigtable.chainId, validatorKey, bigtable.reversedPaddedEpoch(startEpoch), bigtable.reversedPaddedSlot(startEpoch*utils.Config.Chain.Config.SlotsPerEpoch), "\x00") - rangeStart := fmt.Sprintf("%s:%s:%s:%s", bigtable.chainId, validatorKey, bigtable.reversedPaddedEpoch(endEpoch), bigtable.reversedPaddedSlot(endEpoch*utils.Config.Chain.Config.SlotsPerEpoch+utils.Config.Chain.Config.SlotsPerEpoch-1)) + rangeEnd := fmt.Sprintf("%s:%s:%s:%s:%s%s", bigtable.chainId, validatorKey, prefix, bigtable.reversedPaddedEpoch(startEpoch), bigtable.reversedPaddedSlot(startEpoch*utils.Config.Chain.Config.SlotsPerEpoch), "\x00") + rangeStart := fmt.Sprintf("%s:%s:%s:%s:%s", bigtable.chainId, validatorKey, prefix, bigtable.reversedPaddedEpoch(endEpoch), bigtable.reversedPaddedSlot(endEpoch*utils.Config.Chain.Config.SlotsPerEpoch+utils.Config.Chain.Config.SlotsPerEpoch-1)) ranges = append(ranges, gcp_bigtable.NewRange(rangeStart, rangeEnd)) } return ranges } -func (bigtable *Bigtable) getValidatorSlotRanges(validatorIndices []uint64, startSlot uint64, endSlot uint64) gcp_bigtable.RowRangeList { +func (bigtable *Bigtable) getValidatorSlotRanges(validatorIndices []uint64, prefix string, startSlot uint64, endSlot uint64) gcp_bigtable.RowRangeList { if endSlot < startSlot { // handle overflows startSlot = 0 @@ -2350,13 +2156,13 @@ func (bigtable *Bigtable) getValidatorSlotRanges(validatorIndices []uint64, star startEpoch := utils.EpochOfSlot(startSlot) endEpoch := utils.EpochOfSlot(endSlot) - ranges := make(gcp_bigtable.RowRangeList, 0, len(validatorIndices)) + ranges := make(gcp_bigtable.RowRangeList, 0, len(validatorIndices)+1) for _, validatorIndex := range validatorIndices { validatorKey := bigtable.validatorIndexToKey(validatorIndex) - rangeEnd := fmt.Sprintf("%s:%s:%s:%s%s", bigtable.chainId, validatorKey, bigtable.reversedPaddedEpoch(startEpoch), bigtable.reversedPaddedSlot(startSlot), "\x00") - rangeStart := fmt.Sprintf("%s:%s:%s:%s", bigtable.chainId, validatorKey, bigtable.reversedPaddedEpoch(endEpoch), bigtable.reversedPaddedSlot(endSlot)) + rangeEnd := fmt.Sprintf("%s:%s:%s:%s:%s%s", bigtable.chainId, validatorKey, prefix, bigtable.reversedPaddedEpoch(startEpoch), bigtable.reversedPaddedSlot(startSlot), "\x00") + rangeStart := fmt.Sprintf("%s:%s:%s:%s:%s", bigtable.chainId, validatorKey, prefix, bigtable.reversedPaddedEpoch(endEpoch), bigtable.reversedPaddedSlot(endSlot)) ranges = append(ranges, gcp_bigtable.NewRange(rangeStart, rangeEnd)) } diff --git a/db/bigtable_init.go b/db/bigtable_init.go index 56536236d5..022aecef30 100644 --- a/db/bigtable_init.go +++ b/db/bigtable_init.go @@ -13,27 +13,19 @@ func InitBigtableSchema() error { tables := make(map[string]map[string]gcp_bigtable.GCPolicy) - tables["beaconchain_validator_balances"] = map[string]gcp_bigtable.GCPolicy{ - VALIDATOR_BALANCES_FAMILY: nil, - VALIDATOR_HIGHEST_ACTIVE_INDEX_FAMILY: nil, - } - tables["beaconchain_validator_attestations"] = map[string]gcp_bigtable.GCPolicy{ - ATTESTATIONS_FAMILY: nil, - } - tables["beaconchain_validator_proposals"] = map[string]gcp_bigtable.GCPolicy{ - PROPOSALS_FAMILY: nil, - } - tables["beaconchain_validator_sync"] = map[string]gcp_bigtable.GCPolicy{ - SYNC_COMMITTEES_FAMILY: nil, - SYNC_COMMITTEES_PARTICIPATION_FAMILY: nil, - } - tables["beaconchain_validator_income"] = map[string]gcp_bigtable.GCPolicy{ - INCOME_DETAILS_COLUMN_FAMILY: nil, - STATS_COLUMN_FAMILY: nil, - } tables["beaconchain_validators"] = map[string]gcp_bigtable.GCPolicy{ ATTESTATIONS_FAMILY: gcp_bigtable.MaxVersionsGCPolicy(1), } + tables["beaconchain_validators_history"] = map[string]gcp_bigtable.GCPolicy{ + VALIDATOR_BALANCES_FAMILY: nil, + VALIDATOR_HIGHEST_ACTIVE_INDEX_FAMILY: nil, + ATTESTATIONS_FAMILY: nil, + PROPOSALS_FAMILY: nil, + SYNC_COMMITTEES_FAMILY: nil, + SYNC_COMMITTEES_PARTICIPATION_FAMILY: nil, + INCOME_DETAILS_COLUMN_FAMILY: nil, + STATS_COLUMN_FAMILY: nil, + } tables["blocks"] = map[string]gcp_bigtable.GCPolicy{ DEFAULT_FAMILY_BLOCKS: gcp_bigtable.MaxVersionsGCPolicy(1), } diff --git a/db/db.go b/db/db.go index d0575f1260..fd0e1abd69 100644 --- a/db/db.go +++ b/db/db.go @@ -519,8 +519,11 @@ func GetAllEpochs() ([]uint64, error) { // Get latest finalized epoch func GetLatestFinalizedEpoch() (uint64, error) { var latestFinalized uint64 - err := WriterDb.Get(&latestFinalized, "SELECT finalized_epoch FROM chain_head") + err := WriterDb.Get(&latestFinalized, "SELECT epoch FROM epochs WHERE finalized ORDER BY epoch DESC LIMIT 1") if err != nil { + if err == sql.ErrNoRows { + return 0, nil + } utils.LogError(err, "error retrieving latest exported finalized epoch from the database", 0) return 0, err } @@ -738,67 +741,6 @@ func SaveBlock(block *types.Block, forceSlotUpdate bool) error { return nil } -func UpdateChainHead(head *types.ChainHead) error { - count := 0 - err := ReaderDb.Get(&count, "SELECT COUNT(*) FROM chain_head") - if err != nil { - return fmt.Errorf("error getting count from chain_head: %w", err) - } - - query := ` - INSERT INTO chain_head ( - finalized_block_root, - finalized_epoch, - finalized_slot, - head_block_root, - head_epoch, - head_slot, - justified_block_root, - justified_epoch, - justified_slot, - previous_justified_block_root, - previous_justified_epoch, - previous_justified_slot - ) - VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12) - ` - if count > 0 { - query = ` - UPDATE chain_head SET - finalized_block_root = $1, - finalized_epoch = $2, - finalized_slot = $3, - head_block_root = $4, - head_epoch = $5, - head_slot = $6, - justified_block_root = $7, - justified_epoch = $8, - justified_slot = $9, - previous_justified_block_root = $10, - previous_justified_epoch = $11, - previous_justified_slot = $12 - ` - } - _, err = WriterDb.Exec(query, - head.FinalizedBlockRoot, - head.FinalizedEpoch, - head.FinalizedSlot, - head.HeadBlockRoot, - head.HeadEpoch, - head.HeadSlot, - head.JustifiedBlockRoot, - head.JustifiedEpoch, - head.JustifiedSlot, - head.PreviousJustifiedBlockRoot, - head.PreviousJustifiedEpoch, - head.PreviousJustifiedSlot) - if err != nil { - return fmt.Errorf("error updating chain_head: %w", err) - } - - return nil -} - // SaveEpoch will save the epoch data into the database func SaveEpoch(data *types.EpochData, client rpc.Client) error { // Check if we need to export the epoch @@ -847,7 +789,7 @@ func SaveEpoch(data *types.EpochData, client rpc.Client) error { return fmt.Errorf("error saving blocks to db: %w", err) } - if uint64(utils.TimeToEpoch(time.Now())) > data.Epoch+10 { + if data.Epoch%10 != 0 && uint64(utils.TimeToEpoch(time.Now())) > data.Epoch+10 { logger.WithFields(logrus.Fields{"exportEpoch": data.Epoch, "chainEpoch": utils.TimeToEpoch(time.Now())}).Infof("skipping exporting validators because epoch is far behind head") } else { go func() { @@ -942,9 +884,10 @@ func SaveEpoch(data *types.EpochData, client rpc.Client) error { totalvalidatorbalance, eligibleether, globalparticipationrate, - votedether + votedether, + finalized ) - VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15) ON CONFLICT (epoch) DO UPDATE SET blockscount = excluded.blockscount, proposerslashingscount = excluded.proposerslashingscount, @@ -958,7 +901,8 @@ func SaveEpoch(data *types.EpochData, client rpc.Client) error { totalvalidatorbalance = excluded.totalvalidatorbalance, eligibleether = excluded.eligibleether, globalparticipationrate = excluded.globalparticipationrate, - votedether = excluded.votedether`, + votedether = excluded.votedether, + finalized = excluded.finalized`, data.Epoch, len(data.Blocks), proposerSlashingsCount, @@ -972,7 +916,8 @@ func SaveEpoch(data *types.EpochData, client rpc.Client) error { validatorBalanceSum.Uint64(), data.EpochParticipationStats.EligibleEther, data.EpochParticipationStats.GlobalParticipationRate, - data.EpochParticipationStats.VotedEther) + data.EpochParticipationStats.VotedEther, + data.Finalized) if err != nil { return fmt.Errorf("error executing save epoch statement: %w", err) @@ -1820,7 +1765,7 @@ func UpdateEpochStatus(stats *types.ValidatorParticipation) error { return err } -// GetTotalValidatorsCount will return the total-validator-count +// GetValidatorIndices will return the total-validator-indices func GetValidatorIndices() ([]uint64, error) { indices := []uint64{} err := ReaderDb.Select(&indices, "select validatorindex from validators order by validatorindex;") @@ -3309,6 +3254,10 @@ func GetValidatorBalanceForDay(validators []uint64, day uint64, balance *uint64) } func GetValidatorActivationBalance(validators []uint64, balance *uint64) error { + if len(validators) == 0 { + return fmt.Errorf("passing empty validator array is unsupported") + } + validatorsPQArray := pq.Array(validators) return ReaderDb.Get(balance, ` SELECT diff --git a/db/statistics.go b/db/statistics.go index 0e3b37143d..1f487935f5 100644 --- a/db/statistics.go +++ b/db/statistics.go @@ -78,7 +78,7 @@ func WriteValidatorStatisticsForDay(day uint64, concurrencyTotal uint64, concurr } validators := make([]uint64, 0, maxValidatorIndex) - logger.Info(maxValidatorIndex) + logger.Infof("processing statistics for validators 0-%d", maxValidatorIndex) for i := uint64(0); i <= maxValidatorIndex; i++ { validators = append(validators, i) } @@ -87,7 +87,6 @@ func WriteValidatorStatisticsForDay(day uint64, concurrencyTotal uint64, concurr logger.Infof("Skipping day %v as it is already exported", day) return nil } - logger.Info("OK") if exported.FailedAttestations { logger.Infof("Skipping failed attestations") @@ -1127,7 +1126,7 @@ func WriteValidatorFailedAttestationsStatisticsForDay(validators []uint64, day u } g, gCtx := errgroup.WithContext(ctx) - g.SetLimit(50) + g.SetLimit(int(concurrency)) dbBatchSize := 100 for b := 0; b < len(maArr); b += dbBatchSize { diff --git a/exporter/exporter.go b/exporter/exporter.go index f0c57e5d38..5a6f7dd24f 100644 --- a/exporter/exporter.go +++ b/exporter/exporter.go @@ -55,10 +55,6 @@ func Start(client rpc.Client) error { head, err := client.GetChainHead() if err == nil { logger.Infof("Beacon node is available with head slot: %v", head.HeadSlot) - err = db.UpdateChainHead(head) - if err != nil { - utils.LogError(err, "error updating chain head", 0) - } // if we are still waiting for genesis, export epoch 0 with genesis data // epoch 0 will only contain genesis data at this point if head.HeadSlot == 0 { @@ -80,10 +76,6 @@ func Start(client rpc.Client) error { if err != nil { utils.LogFatal(err, "getting chain head from client for full db reindex error", 0) } - err = db.UpdateChainHead(head) - if err != nil { - utils.LogError(err, "error updating chain head", 0) - } for epoch := uint64(0); epoch <= head.HeadEpoch; epoch++ { err := ExportEpoch(epoch, client) @@ -100,10 +92,6 @@ func Start(client rpc.Client) error { if err != nil { utils.LogFatal(err, "getting chain head from client for full canon check error", 0) } - err = db.UpdateChainHead(head) - if err != nil { - utils.LogError(err, "error updating chain head", 0) - } for epoch := int64(head.HeadEpoch) - 1; epoch >= 0; epoch-- { blocks, err := client.GetBlockStatusByEpoch(uint64(epoch)) @@ -121,6 +109,7 @@ func Start(client rpc.Client) error { if utils.Config.Indexer.IndexMissingEpochsOnStartup { // Add any missing epoch to the export set (might happen if the indexer was stopped for a long period of time) + logger.Infof("checking for missing epochs") epochs, err := db.GetAllEpochs() if err != nil { utils.LogFatal(err, "getting all epochs from db error", 0) @@ -156,10 +145,6 @@ func Start(client rpc.Client) error { if err != nil { utils.LogFatal(err, "getting chain head from client for full blocks check error", 0) } - err = db.UpdateChainHead(head) - if err != nil { - utils.LogError(err, "error updating chain head", 0) - } dbBlocks, err := db.GetLastPendingAndProposedBlocks(1, head.HeadEpoch) if err != nil { @@ -237,10 +222,6 @@ func Start(client rpc.Client) error { if err != nil { utils.LogFatal(err, "getting chain head from client for updating epoch statistics error", 0) } - err = db.UpdateChainHead(head) - if err != nil { - utils.LogError(err, "error updating chain head", 0) - } startEpoch := uint64(0) err = updateEpochStatus(client, startEpoch, head.HeadEpoch) if err != nil { @@ -277,11 +258,29 @@ func Start(client rpc.Client) error { } blocksMap[block.Slot][fmt.Sprintf("%x", block.BlockRoot)] = block - err := db.BigtableClient.SaveAttestations(blocksMap) + syncDuties := make(map[types.Slot]map[types.ValidatorIndex]bool) + syncDuties[types.Slot(block.Slot)] = make(map[types.ValidatorIndex]bool) + + for validator, duty := range block.SyncDuties { + syncDuties[types.Slot(block.Slot)][types.ValidatorIndex(validator)] = duty + } + + attDuties := make(map[types.Slot]map[types.ValidatorIndex][]types.Slot) + for validator, attestedSlot := range block.AttestationDuties { + if attDuties[types.Slot(attestedSlot)] == nil { + attDuties[types.Slot(attestedSlot)] = make(map[types.ValidatorIndex][]types.Slot) + } + if attDuties[types.Slot(attestedSlot)][types.ValidatorIndex(validator)] == nil { + attDuties[types.Slot(attestedSlot)][types.ValidatorIndex(validator)] = []types.Slot{} + } + attDuties[types.Slot(attestedSlot)][types.ValidatorIndex(validator)] = append(attDuties[types.Slot(attestedSlot)][types.ValidatorIndex(validator)], types.Slot(block.Slot)) + } + + err := db.BigtableClient.SaveAttestationDuties(attDuties) if err != nil { logrus.Errorf("error exporting attestations to bigtable for block %v: %v", block.Slot, err) } - err = db.BigtableClient.SaveSyncComitteeDuties(blocksMap) + err = db.BigtableClient.SaveSyncComitteeDuties(syncDuties) if err != nil { logrus.Errorf("error exporting sync committee duties to bigtable for block %v: %v", block.Slot, err) } @@ -310,10 +309,6 @@ func doFullCheck(client rpc.Client, lookback uint64) { logger.Errorf("error retrieving chain head: %v", err) return } - err = db.UpdateChainHead(head) - if err != nil { - utils.LogError(err, "error updating chain head", 0) - } startEpoch := uint64(0) // Set the start epoch to the epoch prior to the last finalized epoch @@ -449,6 +444,14 @@ func doFullCheck(client rpc.Client, lookback uint64) { } } logger.Printf("finished export for epoch %v", epoch) + + if len(keys) > 10 && epoch%10 == 0 && epoch >= 10 { // this ensures that epoch finalization is properly updated during long running exports + logger.Infof("updating status of epochs %v-%v", startEpoch, head.HeadEpoch) + err = updateEpochStatus(client, epoch-10, epoch) + if err != nil { + logger.Errorf("error updating epoch stratus: %v", err) + } + } } logger.Infof("marking orphaned blocks of epochs %v-%v", startEpoch, head.HeadEpoch) @@ -576,7 +579,7 @@ func ExportEpoch(epoch uint64, client rpc.Client) error { return nil }) g.Go(func() error { - err = db.BigtableClient.SaveAttestations(data.Blocks) + err = db.BigtableClient.SaveAttestationDuties(data.AttestationDuties) if err != nil { return fmt.Errorf("error exporting attestations to bigtable: %v", err) } @@ -590,7 +593,7 @@ func ExportEpoch(epoch uint64, client rpc.Client) error { return nil }) g.Go(func() error { - err = db.BigtableClient.SaveSyncComitteeDuties(data.Blocks) + err = db.BigtableClient.SaveSyncComitteeDuties(data.SyncDuties) if err != nil { return fmt.Errorf("error exporting sync committee duties to bigtable: %v", err) } @@ -660,10 +663,6 @@ func networkLivenessUpdater(client rpc.Client) { time.Sleep(slotDuration) continue } - err = db.UpdateChainHead(head) - if err != nil { - utils.LogError(err, "error updating chain head", 0) - } // wait for node to be synced if time.Now().Add(-epochDuration).After(utils.EpochToTime(head.HeadEpoch)) { diff --git a/exporter/sync_committees_count.go b/exporter/sync_committees_count.go index 145eb68dc1..b22cb87e31 100644 --- a/exporter/sync_committees_count.go +++ b/exporter/sync_committees_count.go @@ -73,7 +73,7 @@ func exportSyncCommitteesCountAtPeriod(period uint64, countSoFar float64) (float totalValidatorsCount := uint64(0) err := db.WriterDb.Get(&totalValidatorsCount, "SELECT validatorscount FROM epochs WHERE epoch = $1", e) if err != nil { - return 0, err + return 0, fmt.Errorf("error retrieving validatorscount for epoch %v: %v", e, err) } count = countSoFar + (float64(utils.Config.Chain.Config.SyncCommitteeSize) / float64(totalValidatorsCount)) } diff --git a/handlers/api.go b/handlers/api.go index 761444f785..6f60e83e9c 100644 --- a/handlers/api.go +++ b/handlers/api.go @@ -1134,6 +1134,8 @@ func getExpectedSyncCommitteeSlots(validators []uint64, epoch uint64) (expectedS if err != nil { return 0, err } + logger.Info(countStatistics) + logger.Info(periodSlice) if len(countStatistics) != len(periodSlice) { return 0, fmt.Errorf("unable to retrieve all sync committee count statistics, required %v entries but got %v entries (epoch: %v)", len(periodSlice), len(countStatistics), epoch) } @@ -1178,9 +1180,11 @@ func getSyncCommitteeSlotsStatistics(validators []uint64, epoch uint64) (types.S // validator_stats is updated only once a day, everything missing has to be collected from bigtable (which is slower than validator_stats) // check when the last update to validator_stats was epochsPerDay := utils.EpochsPerDay() - lastExportedDay, err := services.LatestExportedStatisticDay() lastExportedEpoch := uint64(0) - if err == nil { + lastExportedDay, err := services.LatestExportedStatisticDay() + if err != nil && err != db.ErrNoStats { + return types.SyncCommitteesStats{}, fmt.Errorf("error retrieving latest exported statistics day: %v", err) + } else if err == nil { lastExportedEpoch = ((lastExportedDay + 1) * epochsPerDay) - 1 } diff --git a/handlers/api_eth1.go b/handlers/api_eth1.go index 6c6e42193f..3dab33536c 100644 --- a/handlers/api_eth1.go +++ b/handlers/api_eth1.go @@ -959,9 +959,11 @@ func getValidatorExecutionPerformance(queryIndices []uint64) ([]types.ExecutionP } } - lastStatsDay, err := services.LatestExportedStatisticDay() firstEpochTime := utils.EpochToTime(0) - if err == nil { + lastStatsDay, err := services.LatestExportedStatisticDay() + if err != nil && err != db.ErrNoStats { + return nil, fmt.Errorf("error retrieving latest exported statistics day: %v", err) + } else if err == nil { firstEpochTime = utils.EpochToTime((lastStatsDay + 1) * utils.EpochsPerDay()) } diff --git a/handlers/validator.go b/handlers/validator.go index 1b8e53df39..83b79a5f02 100644 --- a/handlers/validator.go +++ b/handlers/validator.go @@ -1652,9 +1652,6 @@ func ValidatorHistory(w http.ResponseWriter, r *http.Request) { } tableData = append(tableData, icomeToTableData(epoch, incomeDetails[index][epoch], withdrawalMap[epoch], currency)) - if epoch == 0 { - break - } } } @@ -1910,7 +1907,7 @@ func ValidatorSync(w http.ResponseWriter, r *http.Request) { for slot := firstSlot; slot <= lastSlot; slot++ { if slot > latestProposedSlot { - continue + break } slots = append(slots, slot) } diff --git a/rpc/lighthouse.go b/rpc/lighthouse.go index 1d52ca64eb..fa1e937c34 100644 --- a/rpc/lighthouse.go +++ b/rpc/lighthouse.go @@ -18,10 +18,10 @@ import ( "github.com/donovanhide/eventsource" gtypes "github.com/ethereum/go-ethereum/core/types" + "golang.org/x/sync/errgroup" lru "github.com/hashicorp/golang-lru" "github.com/prysmaticlabs/go-bitfield" - "github.com/sirupsen/logrus" ) // LighthouseLatestHeadEpoch is used to cache the latest head epoch for participation requests @@ -260,11 +260,21 @@ func (lc *LighthouseClient) GetEpochAssignments(epoch uint64) (*types.EpochAssig // GetEpochData will get the epoch data from Lighthouse RPC api func (lc *LighthouseClient) GetEpochData(epoch uint64, skipHistoricBalances bool) (*types.EpochData, error) { - wg := &sync.WaitGroup{} + wg := &errgroup.Group{} mux := &sync.Mutex{} - data := &types.EpochData{} + head, err := lc.GetChainHead() + if err != nil { + return nil, fmt.Errorf("error retrieving chain head: %v", err) + } + data := &types.EpochData{ + SyncDuties: make(map[types.Slot]map[types.ValidatorIndex]bool), + AttestationDuties: make(map[types.Slot]map[types.ValidatorIndex][]types.Slot), + } data.Epoch = epoch + if head.FinalizedEpoch >= epoch { + data.Finalized = true + } validatorsResp, err := lc.get(fmt.Sprintf("%s/eth/v1/beacon/states/%d/validators", lc.endpoint, epoch*utils.Config.Chain.Config.SlotsPerEpoch)) if err != nil && epoch == 0 { @@ -300,52 +310,87 @@ func (lc *LighthouseClient) GetEpochData(epoch uint64, skipHistoricBalances bool logger.Printf("retrieved data for %v validators for epoch %v", len(data.Validators), epoch) - wg.Add(1) - go func() { - defer wg.Done() + wg.Go(func() error { var err error data.ValidatorAssignmentes, err = lc.GetEpochAssignments(epoch) if err != nil { - logrus.Errorf("error retrieving assignments for epoch %v: %v", epoch, err) - return + return fmt.Errorf("error retrieving assignments for epoch %v: %v", epoch, err) } - logger.Printf("retrieved validator assignment data for epoch %v", epoch) - }() - wg.Add(1) - go func() { - defer wg.Done() - data.EpochParticipationStats, err = lc.GetValidatorParticipation(epoch) - if err != nil { - if strings.HasSuffix(err.Error(), "can't be retrieved as it hasn't finished yet") { - logger.Warnf("error retrieving epoch participation statistics for epoch %v: %v", epoch, err) - } else { - logger.Errorf("error retrieving epoch participation statistics for epoch %v: %v", epoch, err) + for slot := epoch * utils.Config.Chain.Config.SlotsPerEpoch; slot <= (epoch+1)*utils.Config.Chain.Config.SlotsPerEpoch-1; slot++ { + if data.SyncDuties[types.Slot(slot)] == nil { + data.SyncDuties[types.Slot(slot)] = make(map[types.ValidatorIndex]bool) } - data.EpochParticipationStats = &types.ValidatorParticipation{ - Epoch: epoch, - GlobalParticipationRate: 1.0, - VotedEther: 0, - EligibleEther: 0, + for _, validatorIndex := range data.ValidatorAssignmentes.SyncAssignments { + data.SyncDuties[types.Slot(slot)][types.ValidatorIndex(validatorIndex)] = false } } - }() + + for key, validatorIndex := range data.ValidatorAssignmentes.AttestorAssignments { + keySplit := strings.Split(key, "-") + attestedSlot, err := strconv.ParseUint(keySplit[0], 10, 64) + + if err != nil { + return fmt.Errorf("error parsing attested slot from attestation key: %v", err) + } + + if data.AttestationDuties[types.Slot(attestedSlot)] == nil { + data.AttestationDuties[types.Slot(attestedSlot)] = make(map[types.ValidatorIndex][]types.Slot) + } + + data.AttestationDuties[types.Slot(attestedSlot)][types.ValidatorIndex(validatorIndex)] = []types.Slot{} + + } + logger.Printf("retrieved validator assignment data for epoch %v", epoch) + return nil + }) + + if epoch < head.HeadEpoch { + wg.Go(func() error { + data.EpochParticipationStats, err = lc.GetValidatorParticipation(epoch) + if err != nil { + if strings.HasSuffix(err.Error(), "can't be retrieved as it hasn't finished yet") { // should no longer happen + logger.Warnf("error retrieving epoch participation statistics for epoch %v: %v", epoch, err) + } else { + return fmt.Errorf("error retrieving epoch participation statistics for epoch %v: %v", epoch, err) + } + data.EpochParticipationStats = &types.ValidatorParticipation{ + Epoch: epoch, + GlobalParticipationRate: 1.0, + VotedEther: 0, + EligibleEther: 0, + } + } + return nil + }) + } else { + data.EpochParticipationStats = &types.ValidatorParticipation{ + Epoch: epoch, + GlobalParticipationRate: 1.0, + VotedEther: 0, + EligibleEther: 0, + } + } + + err = wg.Wait() + if err != nil { + return nil, err + } + wg = &errgroup.Group{} // Retrieve all blocks for the epoch data.Blocks = make(map[uint64]map[string]*types.Block) for slot := epoch * utils.Config.Chain.Config.SlotsPerEpoch; slot <= (epoch+1)*utils.Config.Chain.Config.SlotsPerEpoch-1; slot++ { - if slot != 0 && utils.SlotToTime(slot).After(time.Now()) { // don't export slots that have not occured yet + if slot != 0 && slot > head.HeadSlot { // don't export slots that have not occured yet continue } - wg.Add(1) - go func(slot uint64) { - defer wg.Done() + slot := slot + wg.Go(func() error { blocks, err := lc.GetBlocksBySlot(slot) if err != nil { - logger.Errorf("error retrieving blocks for slot %v: %v", slot, err) - return + return fmt.Errorf("error retrieving blocks for slot %v: %v", slot, err) } for _, block := range blocks { @@ -354,11 +399,63 @@ func (lc *LighthouseClient) GetEpochData(epoch uint64, skipHistoricBalances bool data.Blocks[block.Slot] = make(map[string]*types.Block) } data.Blocks[block.Slot][fmt.Sprintf("%x", block.BlockRoot)] = block + + for validator, duty := range block.SyncDuties { + data.SyncDuties[types.Slot(block.Slot)][types.ValidatorIndex(validator)] = duty + } + for validator, attestedSlot := range block.AttestationDuties { + if data.AttestationDuties[types.Slot(attestedSlot)] == nil { + data.AttestationDuties[types.Slot(attestedSlot)] = make(map[types.ValidatorIndex][]types.Slot) + } + if data.AttestationDuties[types.Slot(attestedSlot)][types.ValidatorIndex(validator)] == nil { + data.AttestationDuties[types.Slot(attestedSlot)][types.ValidatorIndex(validator)] = make([]types.Slot, 0, 10) + } + data.AttestationDuties[types.Slot(attestedSlot)][types.ValidatorIndex(validator)] = append(data.AttestationDuties[types.Slot(attestedSlot)][types.ValidatorIndex(validator)], types.Slot(block.Slot)) + } mux.Unlock() } - }(slot) + return nil + }) } - wg.Wait() + + // we need future blocks to properly tracke fullfilled attestation duties + data.FutureBlocks = make(map[uint64]map[string]*types.Block) + for slot := (epoch + 1) * utils.Config.Chain.Config.SlotsPerEpoch; slot <= (epoch+2)*utils.Config.Chain.Config.SlotsPerEpoch-1; slot++ { + if slot != 0 && slot > head.HeadSlot { // don't export slots that have not occured yet + continue + } + slot := slot + wg.Go(func() error { + blocks, err := lc.GetBlocksBySlot(slot) + + if err != nil { + return fmt.Errorf("error retrieving blocks for slot %v: %v", slot, err) + } + + for _, block := range blocks { + mux.Lock() + if data.FutureBlocks[block.Slot] == nil { + data.FutureBlocks[block.Slot] = make(map[string]*types.Block) + } + data.FutureBlocks[block.Slot][fmt.Sprintf("%x", block.BlockRoot)] = block + + // fill out performed attestation duties + for validator, attestedSlot := range block.AttestationDuties { + if attestedSlot < types.Slot((epoch+1)*utils.Config.Chain.Config.SlotsPerEpoch) { + data.AttestationDuties[types.Slot(attestedSlot)][types.ValidatorIndex(validator)] = append(data.AttestationDuties[types.Slot(attestedSlot)][types.ValidatorIndex(validator)], types.Slot(block.Slot)) + } + } + mux.Unlock() + } + return nil + }) + } + + err = wg.Wait() + if err != nil { + return nil, err + } + logger.Printf("retrieved %v blocks for epoch %v", len(data.Blocks), epoch) if data.ValidatorAssignmentes == nil { @@ -400,29 +497,16 @@ func (lc *LighthouseClient) GetEpochData(epoch uint64, skipHistoricBalances bool // Block is in the past, set status to missed data.Blocks[slot]["0x0"].Status = 2 data.Blocks[slot]["0x0"].BlockRoot = []byte{0x1} - - logger.Infof("backfilling data for missed slot %v", slot) - // Fill sync & attestation assignments - data.Blocks[slot]["0x0"].EmptySlotSyncAssignments = data.ValidatorAssignmentes.SyncAssignments - data.Blocks[slot]["0x0"].EmptySlotAttestationAssignments = make([]uint64, 0, len(data.Validators)/int(utils.Config.Chain.Config.SlotsPerEpoch)) - for key, validator := range data.ValidatorAssignmentes.AttestorAssignments { - keySplit := strings.Split(key, "-") - if len(keySplit) != 3 { - return nil, fmt.Errorf("error parsing slot from validator attestation assignment key %s: invalid format", key) - } - attestedSlot, err := strconv.ParseUint(keySplit[0], 10, 64) - if err != nil { - return nil, fmt.Errorf("error parsing slot from validator attestation assignment key %s: %w", key, err) - } - - if slot == attestedSlot { - data.Blocks[slot]["0x0"].EmptySlotAttestationAssignments = append(data.Blocks[slot]["0x0"].EmptySlotAttestationAssignments, validator) - } - } } } } + // for validator, duty := range data.AttestationDuties { + // for slot, inclusions := range duty { + // logger.Infof("validator %v has attested for slot %v in slots %v", validator, slot, inclusions) + // } + // } + return data, nil } @@ -583,6 +667,8 @@ func (lc *LighthouseClient) blockFromResponse(parsedHeaders *StandardBeaconHeade Deposits: make([]*types.Deposit, len(parsedBlock.Message.Body.Deposits)), VoluntaryExits: make([]*types.VoluntaryExit, len(parsedBlock.Message.Body.VoluntaryExits)), SignedBLSToExecutionChange: make([]*types.SignedBLSToExecutionChange, len(parsedBlock.Message.Body.SignedBLSToExecutionChange)), + AttestationDuties: make(map[types.ValidatorIndex]types.Slot), + SyncDuties: make(map[types.ValidatorIndex]bool), } epochAssignments, err := lc.GetEpochAssignments(slot / utils.Config.Chain.Config.SlotsPerEpoch) @@ -603,6 +689,16 @@ func (lc *LighthouseClient) blockFromResponse(parsedHeaders *StandardBeaconHeade SyncAggregateParticipation: syncCommitteeParticipation(bits), SyncCommitteeSignature: utils.MustParseHex(agg.SyncCommitteeSignature), } + + // fill out performed sync duties + bitLen := len(block.SyncAggregate.SyncCommitteeBits) * 8 + valLen := len(block.SyncAggregate.SyncCommitteeValidators) + if bitLen < valLen { + return nil, fmt.Errorf("error getting sync_committee participants: bitLen != valLen: %v != %v", bitLen, valLen) + } + for i, valIndex := range block.SyncAggregate.SyncCommitteeValidators { + block.SyncDuties[types.ValidatorIndex(valIndex)] = utils.BitAtVector(block.SyncAggregate.SyncCommitteeBits, i) + } } if payload := parsedBlock.Message.Body.ExecutionPayload; payload != nil && !bytes.Equal(payload.ParentHash, make([]byte, 32)) { @@ -763,6 +859,8 @@ func (lc *LighthouseClient) blockFromResponse(parsedHeaders *StandardBeaconHeade logger.Errorf("error retrieving assigned validator for attestation %v of block %v for slot %v committee index %v member index %v", i, block.Slot, a.Data.Slot, a.Data.CommitteeIndex, i) } a.Attesters = append(a.Attesters, validator) + + block.AttestationDuties[types.ValidatorIndex(validator)] = types.Slot(a.Data.Slot) } } diff --git a/types/exporter.go b/types/exporter.go index c45fad2013..1ab32f8461 100644 --- a/types/exporter.go +++ b/types/exporter.go @@ -45,13 +45,20 @@ type FinalityCheckpoints struct { } `json:"finalized"` } +type Slot uint64 +type ValidatorIndex uint64 + // EpochData is a struct to hold epoch data type EpochData struct { Epoch uint64 Validators []*Validator ValidatorAssignmentes *EpochAssignments Blocks map[uint64]map[string]*Block + FutureBlocks map[uint64]map[string]*Block EpochParticipationStats *ValidatorParticipation + AttestationDuties map[Slot]map[ValidatorIndex][]Slot + SyncDuties map[Slot]map[ValidatorIndex]bool + Finalized bool } // ValidatorParticipation is a struct to hold validator participation data @@ -103,28 +110,28 @@ type SyncAggregate struct { // Block is a struct to hold block data type Block struct { - Status uint64 - Proposer uint64 - BlockRoot []byte - Slot uint64 - ParentRoot []byte - StateRoot []byte - Signature []byte - RandaoReveal []byte - Graffiti []byte - Eth1Data *Eth1Data - BodyRoot []byte - ProposerSlashings []*ProposerSlashing - AttesterSlashings []*AttesterSlashing - Attestations []*Attestation - Deposits []*Deposit - VoluntaryExits []*VoluntaryExit - SyncAggregate *SyncAggregate // warning: sync aggregate may be nil, for phase0 blocks - ExecutionPayload *ExecutionPayload // warning: payload may be nil, for phase0/altair blocks - Canonical bool - SignedBLSToExecutionChange []*SignedBLSToExecutionChange - EmptySlotAttestationAssignments []uint64 - EmptySlotSyncAssignments []uint64 + Status uint64 + Proposer uint64 + BlockRoot []byte + Slot uint64 + ParentRoot []byte + StateRoot []byte + Signature []byte + RandaoReveal []byte + Graffiti []byte + Eth1Data *Eth1Data + BodyRoot []byte + ProposerSlashings []*ProposerSlashing + AttesterSlashings []*AttesterSlashing + Attestations []*Attestation + Deposits []*Deposit + VoluntaryExits []*VoluntaryExit + SyncAggregate *SyncAggregate // warning: sync aggregate may be nil, for phase0 blocks + ExecutionPayload *ExecutionPayload // warning: payload may be nil, for phase0/altair blocks + Canonical bool + SignedBLSToExecutionChange []*SignedBLSToExecutionChange + AttestationDuties map[ValidatorIndex]Slot + SyncDuties map[ValidatorIndex]bool } type SignedBLSToExecutionChange struct { From 2e844ebf14eafb2c8fc3501bc69d6b4b2b49c82e Mon Sep 17 00:00:00 2001 From: peter <1674920+peterbitfly@users.noreply.github.com> Date: Sat, 9 Sep 2023 15:48:01 +0200 Subject: [PATCH 081/106] (BIDS-2379) fix build error --- db/bigtable.go | 1 - 1 file changed, 1 deletion(-) diff --git a/db/bigtable.go b/db/bigtable.go index e0291c1d2e..38fcac1bf1 100644 --- a/db/bigtable.go +++ b/db/bigtable.go @@ -1861,7 +1861,6 @@ func (bigtable *Bigtable) MigrateEpochSchemaV1ToV2(epoch uint64) error { mutsInclusionSlot = make([]*gcp_bigtable.Mutation, 0, MAX_BATCH_MUTATIONS) keysInclusionSlot = make([]string, 0, MAX_BATCH_MUTATIONS) } - i++ } if len(mutsInclusionSlot) > 0 { From a3fb1f1abe9d7c550035f88454ddd246f7154935 Mon Sep 17 00:00:00 2001 From: peter <1674920+peterbitfly@users.noreply.github.com> Date: Sat, 9 Sep 2023 15:52:27 +0200 Subject: [PATCH 082/106] (BIDS-2379) use latest finalized epoch to derive the current sync period --- exporter/sync_committees_count.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/exporter/sync_committees_count.go b/exporter/sync_committees_count.go index b22cb87e31..05787f12fc 100644 --- a/exporter/sync_committees_count.go +++ b/exporter/sync_committees_count.go @@ -2,6 +2,7 @@ package exporter import ( "eth2-exporter/db" + "eth2-exporter/services" "eth2-exporter/utils" "fmt" "time" @@ -26,8 +27,7 @@ func exportSyncCommitteesCount() error { return err } - currEpoch := utils.TimeToEpoch(time.Now()) - currentPeriod := utils.SyncPeriodOfEpoch(uint64(currEpoch)) + currentPeriod := utils.SyncPeriodOfEpoch(services.LatestFinalizedEpoch()) firstPeriod := utils.SyncPeriodOfEpoch(utils.Config.Chain.Config.AltairForkEpoch) dbPeriod := uint64(0) From e7affcb51bd1e4114550c4dcf0a9893a0e2647a8 Mon Sep 17 00:00:00 2001 From: peter <1674920+peterbitfly@users.noreply.github.com> Date: Sat, 9 Sep 2023 15:56:55 +0200 Subject: [PATCH 083/106] (BIDS-2379) remove leftover log statements --- handlers/api.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/handlers/api.go b/handlers/api.go index 6f60e83e9c..07a52ee864 100644 --- a/handlers/api.go +++ b/handlers/api.go @@ -1134,8 +1134,6 @@ func getExpectedSyncCommitteeSlots(validators []uint64, epoch uint64) (expectedS if err != nil { return 0, err } - logger.Info(countStatistics) - logger.Info(periodSlice) if len(countStatistics) != len(periodSlice) { return 0, fmt.Errorf("unable to retrieve all sync committee count statistics, required %v entries but got %v entries (epoch: %v)", len(periodSlice), len(countStatistics), epoch) } From be7299f307e0e8cef0ba501576b84cd12dad7b9b Mon Sep 17 00:00:00 2001 From: peter <1674920+peterbitfly@users.noreply.github.com> Date: Sat, 9 Sep 2023 18:53:39 +0200 Subject: [PATCH 084/106] (BIDS-2379) fix sync duties table --- db/bigtable.go | 26 +++++++++++++------------- handlers/validator.go | 15 ++++----------- 2 files changed, 17 insertions(+), 24 deletions(-) diff --git a/db/bigtable.go b/db/bigtable.go index 38fcac1bf1..3a013478b9 100644 --- a/db/bigtable.go +++ b/db/bigtable.go @@ -1215,21 +1215,21 @@ func (bigtable *Bigtable) GetValidatorSyncDutiesHistory(validators []uint64, sta err := bigtable.tableValidatorsHistory.ReadRows(ctx, ranges, func(r gcp_bigtable.Row) bool { - for _, ri := range r[SYNC_COMMITTEES_FAMILY] { - keySplit := strings.Split(r.Key(), ":") + keySplit := strings.Split(r.Key(), ":") - validator, err := bigtable.validatorKeyToIndex(keySplit[1]) - if err != nil { - logger.Errorf("error parsing validator from row key %v: %v", r.Key(), err) - return false - } + validator, err := bigtable.validatorKeyToIndex(keySplit[1]) + if err != nil { + logger.Errorf("error parsing validator from row key %v: %v", r.Key(), err) + return false + } + slot, err := strconv.ParseUint(keySplit[4], 10, 64) + if err != nil { + logger.Errorf("error parsing slot from row key %v: %v", r.Key(), err) + return false + } + slot = MAX_BLOCK_NUMBER - slot + for _, ri := range r[SYNC_COMMITTEES_FAMILY] { - slot, err := strconv.ParseUint(keySplit[3], 10, 64) - if err != nil { - logger.Errorf("error parsing slot from row key %v: %v", r.Key(), err) - return false - } - slot = MAX_BLOCK_NUMBER - slot inclusionSlot := MAX_BLOCK_NUMBER - uint64(ri.Timestamp)/1000 status := uint64(1) // 1: participated diff --git a/handlers/validator.go b/handlers/validator.go index 83b79a5f02..32593f0a1b 100644 --- a/handlers/validator.go +++ b/handlers/validator.go @@ -1871,7 +1871,7 @@ func ValidatorSync(w http.ResponseWriter, r *http.Request) { if length > 100 { length = 100 } - descOrdering := q.Get("order[0][dir]") == "desc" + // descOrdering := q.Get("order[0][dir]") == "desc" // retrieve all sync periods for this validator var syncPeriods []uint64 = []uint64{} @@ -1914,9 +1914,6 @@ func ValidatorSync(w http.ResponseWriter, r *http.Request) { } // if ordering is desc, reverse sync slots - if descOrdering { - utils.ReverseSlice(slots) - } totalCount = uint64(len(slots)) startIndex := start + length - 1 @@ -1925,14 +1922,10 @@ func ValidatorSync(w http.ResponseWriter, r *http.Request) { } endIndex := start - startSlot := slots[startIndex] - - endSlot := slots[endIndex] - - if startSlot > endSlot { // guard against overflow - startSlot = 0 - } + endSlot := slots[startIndex] + startSlot := slots[endIndex] + logger.Infof("retrieving sync duty history for validator %v and slots %v-%v (%v-%v)", validatorIndex, startSlot, endSlot, startIndex, endIndex) syncDuties, err := db.BigtableClient.GetValidatorSyncDutiesHistory([]uint64{validatorIndex}, startSlot, endSlot) if err != nil { From 134e1320aa26e69ebdbc9d29887b21a83c81adf4 Mon Sep 17 00:00:00 2001 From: peter <1674920+peterbitfly@users.noreply.github.com> Date: Sat, 9 Sep 2023 18:54:44 +0200 Subject: [PATCH 085/106] (BIDS-2379) remove log message --- handlers/validator.go | 1 - 1 file changed, 1 deletion(-) diff --git a/handlers/validator.go b/handlers/validator.go index 32593f0a1b..688255086c 100644 --- a/handlers/validator.go +++ b/handlers/validator.go @@ -1925,7 +1925,6 @@ func ValidatorSync(w http.ResponseWriter, r *http.Request) { endSlot := slots[startIndex] startSlot := slots[endIndex] - logger.Infof("retrieving sync duty history for validator %v and slots %v-%v (%v-%v)", validatorIndex, startSlot, endSlot, startIndex, endIndex) syncDuties, err := db.BigtableClient.GetValidatorSyncDutiesHistory([]uint64{validatorIndex}, startSlot, endSlot) if err != nil { From 296d44bf846b351b298980b5b5ac0830c68fb73b Mon Sep 17 00:00:00 2001 From: peter <1674920+peterbitfly@users.noreply.github.com> Date: Mon, 11 Sep 2023 08:37:29 +0200 Subject: [PATCH 086/106] (BIDS-2379) fix epoch finalization & sync committee page issues --- db/db.go | 7 ++++--- exporter/eth1.go | 16 +++++++++------- handlers/validator.go | 14 ++++++++------ rpc/lighthouse.go | 12 ++++++++---- types/exporter.go | 1 + 5 files changed, 30 insertions(+), 20 deletions(-) diff --git a/db/db.go b/db/db.go index fd0e1abd69..22e6f17603 100644 --- a/db/db.go +++ b/db/db.go @@ -1758,9 +1758,10 @@ func UpdateEpochStatus(stats *types.ValidatorParticipation) error { UPDATE epochs SET eligibleether = $1, globalparticipationrate = $2, - votedether = $3 - WHERE epoch = $4`, - stats.EligibleEther, stats.GlobalParticipationRate, stats.VotedEther, stats.Epoch) + votedether = $3, + finalized = $4 + WHERE epoch = $5`, + stats.EligibleEther, stats.GlobalParticipationRate, stats.VotedEther, stats.Finalized, stats.Epoch) return err } diff --git a/exporter/eth1.go b/exporter/eth1.go index 392359ea31..0c89d6d793 100644 --- a/exporter/eth1.go +++ b/exporter/eth1.go @@ -135,13 +135,15 @@ func eth1DepositsExporter() { // make sure we are progressing even if there are no deposits in the last batch lastFetchedBlock = toBlock - logger.WithFields(logrus.Fields{ - "duration": time.Since(t0), - "blockHeight": blockHeight, - "fromBlock": fromBlock, - "toBlock": toBlock, - "depositsSaved": len(depositsToSave), - }).Info("exported eth1-deposits") + if len(depositsToSave) > 0 { + logger.WithFields(logrus.Fields{ + "duration": time.Since(t0), + "blockHeight": blockHeight, + "fromBlock": fromBlock, + "toBlock": toBlock, + "depositsSaved": len(depositsToSave), + }).Info("exported eth1-deposits") + } // progress faster if we are not synced to head yet if blockHeight != toBlock { diff --git a/handlers/validator.go b/handlers/validator.go index 688255086c..3a4ca82a97 100644 --- a/handlers/validator.go +++ b/handlers/validator.go @@ -1903,11 +1903,12 @@ func ValidatorSync(w http.ResponseWriter, r *http.Request) { lastEpoch := firstEpoch + utils.Config.Chain.Config.EpochsPerSyncCommitteePeriod - 1 firstSlot := firstEpoch * utils.Config.Chain.Config.SlotsPerEpoch - lastSlot := lastEpoch*utils.Config.Chain.Config.SlotsPerEpoch - 1 + lastSlot := (lastEpoch+1)*utils.Config.Chain.Config.SlotsPerEpoch - 1 - for slot := firstSlot; slot <= lastSlot; slot++ { - if slot > latestProposedSlot { - break + // logger.Infof("processing sync period %v epoch (%v/%v) slots (%v/%v)", period, firstEpoch, lastEpoch, firstSlot, lastSlot) + for slot := lastSlot; slot >= firstSlot && (slot <= lastSlot /* guards against underflows */); slot-- { + if slot > latestProposedSlot || utils.EpochOfSlot(slot) < utils.Config.Chain.Config.AltairForkEpoch { + continue } slots = append(slots, slot) } @@ -1922,9 +1923,10 @@ func ValidatorSync(w http.ResponseWriter, r *http.Request) { } endIndex := start - endSlot := slots[startIndex] - startSlot := slots[endIndex] + endSlot := slots[endIndex] + startSlot := slots[startIndex] + // logger.Infof("retrieving sync duty history for validator %v and slots %v-%v (%v-%v)", validatorIndex, startSlot, endSlot, startIndex, endIndex) syncDuties, err := db.BigtableClient.GetValidatorSyncDutiesHistory([]uint64{validatorIndex}, startSlot, endSlot) if err != nil { diff --git a/rpc/lighthouse.go b/rpc/lighthouse.go index fa1e937c34..037be9a927 100644 --- a/rpc/lighthouse.go +++ b/rpc/lighthouse.go @@ -913,13 +913,15 @@ func syncCommitteeParticipation(bits []byte) float64 { // GetValidatorParticipation will get the validator participation from the Lighthouse RPC api func (lc *LighthouseClient) GetValidatorParticipation(epoch uint64) (*types.ValidatorParticipation, error) { + + head, err := lc.GetChainHead() + if err != nil { + return nil, err + } + if LighthouseLatestHeadEpoch == 0 || epoch >= LighthouseLatestHeadEpoch-1 { // update LighthouseLatestHeadEpoch to make sure we are continuing with the latest data // we need to check when epoch = head and epoch head - 1 so our following logic acts correctly when we are close to the head - head, err := lc.GetChainHead() - if err != nil { - return nil, err - } logger.Infof("Updating LighthouseLatestHeadEpoch to %v", head.HeadEpoch) LighthouseLatestHeadEpoch = head.HeadEpoch } @@ -962,6 +964,7 @@ func (lc *LighthouseClient) GetValidatorParticipation(epoch uint64) (*types.Vali GlobalParticipationRate: float32(parsedResponse.Data.PreviousEpochTargetAttestingGwei) / float32(parsedResponse.Data.PreviousEpochActiveGwei), VotedEther: uint64(parsedResponse.Data.PreviousEpochTargetAttestingGwei), EligibleEther: uint64(parsedResponse.Data.PreviousEpochActiveGwei), + Finalized: epoch <= head.FinalizedEpoch, } } else { res = &types.ValidatorParticipation{ @@ -969,6 +972,7 @@ func (lc *LighthouseClient) GetValidatorParticipation(epoch uint64) (*types.Vali GlobalParticipationRate: float32(parsedResponse.Data.CurrentEpochTargetAttestingGwei) / float32(parsedResponse.Data.CurrentEpochActiveGwei), VotedEther: uint64(parsedResponse.Data.CurrentEpochTargetAttestingGwei), EligibleEther: uint64(parsedResponse.Data.CurrentEpochActiveGwei), + Finalized: epoch <= head.FinalizedEpoch, } } return res, nil diff --git a/types/exporter.go b/types/exporter.go index 1ab32f8461..9b23abdc17 100644 --- a/types/exporter.go +++ b/types/exporter.go @@ -67,6 +67,7 @@ type ValidatorParticipation struct { GlobalParticipationRate float32 VotedEther uint64 EligibleEther uint64 + Finalized bool } // BeaconCommitteItem is a struct to hold beacon committee data From e91117247ab37bf8ef6cfc4d14dbe333a9bbad46 Mon Sep 17 00:00:00 2001 From: peter <1674920+peterbitfly@users.noreply.github.com> Date: Mon, 11 Sep 2023 08:39:14 +0200 Subject: [PATCH 087/106] (BIDS-2379) disable ordering of the sync duties table --- templates/validator/tables.html | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/templates/validator/tables.html b/templates/validator/tables.html index 7f34d6dc94..be0c289ee0 100644 --- a/templates/validator/tables.html +++ b/templates/validator/tables.html @@ -267,7 +267,7 @@ $('#sync-table').DataTable({ processing: true, serverSide: true, - ordering: true, + ordering: false, order: [[0, 'desc']], lengthChange: false, stateSave: true, @@ -294,17 +294,17 @@ { targets: 0, data: '0', - "orderable": true + "orderable": false }, { targets: 1, data: '1', - "orderable": true + "orderable": false }, { targets: 2, data: '2', - "orderable": true + "orderable": false }, { targets: 3, From b01ffb99cef70cc7603ce9f4a7824eb15a2799ab Mon Sep 17 00:00:00 2001 From: peter <1674920+peterbitfly@users.noreply.github.com> Date: Mon, 11 Sep 2023 08:41:51 +0200 Subject: [PATCH 088/106] (BIDS-2379) support reading config file location from an env variable --- utils/utils.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/utils/utils.go b/utils/utils.go index ac2ebb54cc..6ec467658a 100644 --- a/utils/utils.go +++ b/utils/utils.go @@ -374,6 +374,11 @@ func WaitForCtrlC() { // ReadConfig will process a configuration func ReadConfig(cfg *types.Config, path string) error { + configPathFromEnv := os.Getenv("BEACONCHAIN_CONFIG") + + if configPathFromEnv != "" { // allow the location of the config file to be passed via env args + path = configPathFromEnv + } if strings.HasPrefix(path, "projects/") { x, err := AccessSecretVersion(path) if err != nil { From 5ca091188bdb58bb86886ab8b941f6ee378b313e Mon Sep 17 00:00:00 2001 From: peter <1674920+peterbitfly@users.noreply.github.com> Date: Mon, 11 Sep 2023 09:28:53 +0200 Subject: [PATCH 089/106] (BIDS-2379) fix build errors --- cmd/migrations/bigtable/main.go | 96 ++++++++++++++------------------- 1 file changed, 40 insertions(+), 56 deletions(-) diff --git a/cmd/migrations/bigtable/main.go b/cmd/migrations/bigtable/main.go index 8e004cc753..6c5a7696fe 100644 --- a/cmd/migrations/bigtable/main.go +++ b/cmd/migrations/bigtable/main.go @@ -9,7 +9,6 @@ import ( "flag" "fmt" "math/big" - "strconv" "time" "github.com/sirupsen/logrus" @@ -66,31 +65,6 @@ func main() { utils.LogFatal(err, "deleting epoch error", 0) } - firstSlot := i * utils.Config.Chain.Config.SlotsPerEpoch - lastSlot := (i+1)*utils.Config.Chain.Config.SlotsPerEpoch - 1 - - c, err := rpcClient.GetSyncCommittee(fmt.Sprintf("%d", firstSlot), i) - if err != nil { - utils.LogFatal(err, "getting sync comittee error", 0) - } - - validatorsU64 := make([]uint64, len(c.Validators)) - for i, idxStr := range c.Validators { - idxU64, err := strconv.ParseUint(idxStr, 10, 64) - if err != nil { - utils.LogFatal(err, "parsing validator index to uint error", 0) - } - validatorsU64[i] = idxU64 - } - - logrus.Infof("saving sync assignments for %v validators", len(validatorsU64)) - - err = db.BigtableClient.SaveSyncCommitteesAssignments(firstSlot, lastSlot, validatorsU64) - if err != nil { - logrus.Fatalf("error saving sync committee assignments: %v", err) - } - logrus.Infof("exported sync committee assignments to bigtable in %v", i) - data, err := rpcClient.GetEpochData(uint64(i), true) if err != nil { utils.LogFatal(err, "getting epoch data error", 0) @@ -99,27 +73,32 @@ func main() { g := new(errgroup.Group) g.Go(func() error { - return bt.SaveValidatorBalances(data.Epoch, data.Validators) - }) - - g.Go(func() error { - return bt.SaveAttestationAssignments(data.Epoch, data.ValidatorAssignmentes.AttestorAssignments) - }) - - g.Go(func() error { - return bt.SaveProposalAssignments(data.Epoch, data.ValidatorAssignmentes.ProposerAssignments) + err = db.BigtableClient.SaveProposalAssignments(i, data.ValidatorAssignmentes.ProposerAssignments) + if err != nil { + return fmt.Errorf("error exporting proposal assignments to bigtable: %v", err) + } + return nil }) - g.Go(func() error { - return bt.SaveAttestations(data.Blocks) + err = db.BigtableClient.SaveAttestationDuties(data.AttestationDuties) + if err != nil { + return fmt.Errorf("error exporting attestations to bigtable: %v", err) + } + return nil }) - g.Go(func() error { - return bt.SaveProposals(data.Blocks) + err = db.BigtableClient.SaveProposals(data.Blocks) + if err != nil { + return fmt.Errorf("error exporting proposals to bigtable: %v", err) + } + return nil }) - g.Go(func() error { - return bt.SaveSyncComitteeDuties(data.Blocks) + err = db.BigtableClient.SaveSyncComitteeDuties(data.SyncDuties) + if err != nil { + return fmt.Errorf("error exporting sync committee duties to bigtable: %v", err) + } + return nil }) err = g.Wait() @@ -177,27 +156,32 @@ func monitor(configPath string) { g := new(errgroup.Group) g.Go(func() error { - return bt.SaveValidatorBalances(data.Epoch, data.Validators) + err = db.BigtableClient.SaveProposalAssignments(i, data.ValidatorAssignmentes.ProposerAssignments) + if err != nil { + return fmt.Errorf("error exporting proposal assignments to bigtable: %v", err) + } + return nil }) - g.Go(func() error { - return bt.SaveAttestationAssignments(data.Epoch, data.ValidatorAssignmentes.AttestorAssignments) + err = db.BigtableClient.SaveAttestationDuties(data.AttestationDuties) + if err != nil { + return fmt.Errorf("error exporting attestations to bigtable: %v", err) + } + return nil }) - - g.Go(func() error { - return bt.SaveProposalAssignments(data.Epoch, data.ValidatorAssignmentes.ProposerAssignments) - }) - g.Go(func() error { - return bt.SaveAttestations(data.Blocks) + err = db.BigtableClient.SaveProposals(data.Blocks) + if err != nil { + return fmt.Errorf("error exporting proposals to bigtable: %v", err) + } + return nil }) - - g.Go(func() error { - return bt.SaveProposals(data.Blocks) - }) - g.Go(func() error { - return bt.SaveSyncComitteeDuties(data.Blocks) + err = db.BigtableClient.SaveSyncComitteeDuties(data.SyncDuties) + if err != nil { + return fmt.Errorf("error exporting sync committee duties to bigtable: %v", err) + } + return nil }) err = g.Wait() From 26a446281a573370136371bac56b763cf07c9596 Mon Sep 17 00:00:00 2001 From: peter <1674920+peterbitfly@users.noreply.github.com> Date: Mon, 11 Sep 2023 09:55:21 +0200 Subject: [PATCH 090/106] (BIDS-2379) fix linter --- templates/deposits.html | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/templates/deposits.html b/templates/deposits.html index 75b1928a06..8430ad909b 100644 --- a/templates/deposits.html +++ b/templates/deposits.html @@ -183,7 +183,11 @@

Runner U - {{ if gt (len .Stats.TopDepositors) 1 }}0x{{ (index .Stats.TopDepositors 1).Address }}{{ else }}N/A{{ end }} + {{ if gt (len .Stats.TopDepositors) 1 }} + 0x{{ (index .Stats.TopDepositors 1).Address }} + {{ else }} + N/A + {{ end }}

From 911181f6e357bd1edf42413be7fe894c55630659 Mon Sep 17 00:00:00 2001 From: peter <1674920+peterbitfly@users.noreply.github.com> Date: Mon, 11 Sep 2023 10:34:39 +0200 Subject: [PATCH 091/106] (BIDS-2379) add instrumentation for bigtable calls, fix excessive bigtable data usage for validator sync committee stats --- db/bigtable.go | 488 +++++++++++++++--------------------------- db/bigtable_eth1.go | 375 ++++++++++++++++++++++++++++++++ handlers/validator.go | 5 +- utils/utils.go | 5 + 4 files changed, 554 insertions(+), 319 deletions(-) diff --git a/db/bigtable.go b/db/bigtable.go index 1b874c7531..2613f28ec6 100644 --- a/db/bigtable.go +++ b/db/bigtable.go @@ -43,6 +43,8 @@ const ( MAX_EPOCH = 1000000000 - 1 MAX_BATCH_MUTATIONS = 100000 + + REPORT_TIMEOUT = time.Second * 10 ) type Bigtable struct { @@ -202,6 +204,15 @@ func (bigtable Bigtable) getMachineMetricNamesMap(userID uint64, searchDepth int } func (bigtable Bigtable) GetMachineMetricsMachineNames(userID uint64) ([]string, error) { + tmr := time.NewTimer(REPORT_TIMEOUT) + defer tmr.Stop() + go func() { + <-tmr.C + logger.WithFields(logrus.Fields{ + "userId": userID, + }).Errorf("%s call took longer than %v", utils.GetCurrentFuncName(), REPORT_TIMEOUT) + }() + names, err := bigtable.getMachineMetricNamesMap(userID, 300) if err != nil { return nil, err @@ -216,6 +227,15 @@ func (bigtable Bigtable) GetMachineMetricsMachineNames(userID uint64) ([]string, } func (bigtable Bigtable) GetMachineMetricsMachineCount(userID uint64) (uint64, error) { + tmr := time.NewTimer(REPORT_TIMEOUT) + defer tmr.Stop() + go func() { + <-tmr.C + logger.WithFields(logrus.Fields{ + "userId": userID, + }).Errorf("%s call took longer than %v", utils.GetCurrentFuncName(), REPORT_TIMEOUT) + }() + ctx, cancel := context.WithTimeout(context.Background(), time.Second*10) defer cancel() @@ -229,6 +249,17 @@ func (bigtable Bigtable) GetMachineMetricsMachineCount(userID uint64) (uint64, e } func (bigtable Bigtable) GetMachineMetricsNode(userID uint64, limit, offset int) ([]*types.MachineMetricNode, error) { + tmr := time.NewTimer(REPORT_TIMEOUT) + defer tmr.Stop() + go func() { + <-tmr.C + logger.WithFields(logrus.Fields{ + "userId": userID, + "limit": limit, + "offset": offset, + }).Errorf("%s call took longer than %v", utils.GetCurrentFuncName(), REPORT_TIMEOUT) + }() + return getMachineMetrics(bigtable, "beaconnode", userID, limit, offset, func(data []byte, machine string) *types.MachineMetricNode { obj := &types.MachineMetricNode{} @@ -243,6 +274,17 @@ func (bigtable Bigtable) GetMachineMetricsNode(userID uint64, limit, offset int) } func (bigtable Bigtable) GetMachineMetricsValidator(userID uint64, limit, offset int) ([]*types.MachineMetricValidator, error) { + tmr := time.NewTimer(REPORT_TIMEOUT) + defer tmr.Stop() + go func() { + <-tmr.C + logger.WithFields(logrus.Fields{ + "userId": userID, + "limit": limit, + "offset": offset, + }).Errorf("%s call took longer than %v", utils.GetCurrentFuncName(), REPORT_TIMEOUT) + }() + return getMachineMetrics(bigtable, "validator", userID, limit, offset, func(data []byte, machine string) *types.MachineMetricValidator { obj := &types.MachineMetricValidator{} @@ -257,6 +299,17 @@ func (bigtable Bigtable) GetMachineMetricsValidator(userID uint64, limit, offset } func (bigtable Bigtable) GetMachineMetricsSystem(userID uint64, limit, offset int) ([]*types.MachineMetricSystem, error) { + tmr := time.NewTimer(REPORT_TIMEOUT) + defer tmr.Stop() + go func() { + <-tmr.C + logger.WithFields(logrus.Fields{ + "userId": userID, + "limit": limit, + "offset": offset, + }).Errorf("%s call took longer than %v", utils.GetCurrentFuncName(), REPORT_TIMEOUT) + }() + return getMachineMetrics(bigtable, "system", userID, limit, offset, func(data []byte, machine string) *types.MachineMetricSystem { obj := &types.MachineMetricSystem{} @@ -323,6 +376,15 @@ func (bigtable Bigtable) GetMachineRowKey(userID uint64, process string, machine // and 5 minute old data in fiveMinuteOldData (defined in limit) // as well as the insert timestamps of both func (bigtable Bigtable) GetMachineMetricsForNotifications(rowKeys gcp_bigtable.RowList) (map[uint64]map[string]*types.MachineMetricSystemUser, error) { + tmr := time.NewTimer(REPORT_TIMEOUT) + defer tmr.Stop() + go func() { + <-tmr.C + logger.WithFields(logrus.Fields{ + "rowKeys": rowKeys, + }).Errorf("%s call took longer than %v", utils.GetCurrentFuncName(), REPORT_TIMEOUT) + }() + ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(time.Second*200)) defer cancel() @@ -762,6 +824,14 @@ func (bigtable *Bigtable) SaveSyncComitteeDuties(duties map[types.Slot]map[types // GetMaxValidatorindexForEpoch returns the higest validatorindex with a balance at that epoch func (bigtable *Bigtable) GetMaxValidatorindexForEpoch(epoch uint64) (uint64, error) { + tmr := time.NewTimer(REPORT_TIMEOUT) + defer tmr.Stop() + go func() { + <-tmr.C + logger.WithFields(logrus.Fields{ + "epoch": epoch, + }).Errorf("%s call took longer than %v", utils.GetCurrentFuncName(), REPORT_TIMEOUT) + }() ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(time.Minute*5)) defer cancel() @@ -781,6 +851,16 @@ func (bigtable *Bigtable) GetMaxValidatorindexForEpoch(epoch uint64) (uint64, er } func (bigtable *Bigtable) GetValidatorBalanceHistory(validators []uint64, startEpoch uint64, endEpoch uint64) (map[uint64][]*types.ValidatorBalance, error) { + tmr := time.NewTimer(REPORT_TIMEOUT) + defer tmr.Stop() + go func() { + <-tmr.C + logger.WithFields(logrus.Fields{ + "validators": validators, + "startEpoch": startEpoch, + "endEpoch": endEpoch, + }).Errorf("%s call took longer than %v", utils.GetCurrentFuncName(), REPORT_TIMEOUT) + }() if len(validators) == 0 { return nil, fmt.Errorf("passing empty validator array is unsupported") @@ -875,6 +955,16 @@ func (bigtable *Bigtable) GetValidatorBalanceHistory(validators []uint64, startE } func (bigtable *Bigtable) GetValidatorAttestationHistory(validators []uint64, startEpoch uint64, endEpoch uint64) (map[uint64][]*types.ValidatorAttestation, error) { + tmr := time.NewTimer(REPORT_TIMEOUT) + defer tmr.Stop() + go func() { + <-tmr.C + logger.WithFields(logrus.Fields{ + "validators": validators, + "startEpoch": startEpoch, + "endEpoch": endEpoch, + }).Errorf("%s call took longer than %v", utils.GetCurrentFuncName(), REPORT_TIMEOUT) + }() if len(validators) == 0 { return nil, fmt.Errorf("passing empty validator array is unsupported") @@ -1037,6 +1127,15 @@ func (bigtable *Bigtable) GetValidatorAttestationHistory(validators []uint64, st } func (bigtable *Bigtable) GetLastAttestationSlots(validators []uint64) (map[uint64]uint64, error) { + tmr := time.NewTimer(REPORT_TIMEOUT) + defer tmr.Stop() + go func() { + <-tmr.C + logger.WithFields(logrus.Fields{ + "validators": validators, + }).Errorf("%s call took longer than %v", utils.GetCurrentFuncName(), REPORT_TIMEOUT) + }() + valLen := len(validators) ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(time.Minute*5)) @@ -1093,6 +1192,15 @@ func (bigtable *Bigtable) GetLastAttestationSlots(validators []uint64) (map[uint } func (bigtable *Bigtable) GetSyncParticipationBySlotRange(startSlot, endSlot uint64) (map[uint64]uint64, error) { + tmr := time.NewTimer(REPORT_TIMEOUT) + defer tmr.Stop() + go func() { + <-tmr.C + logger.WithFields(logrus.Fields{ + "startSlot": startSlot, + "endSlot": endSlot, + }).Errorf("%s call took longer than %v", utils.GetCurrentFuncName(), REPORT_TIMEOUT) + }() ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(time.Minute*5)) defer cancel() @@ -1126,6 +1234,16 @@ func (bigtable *Bigtable) GetSyncParticipationBySlotRange(startSlot, endSlot uin } func (bigtable *Bigtable) GetValidatorMissedAttestationHistory(validators []uint64, startEpoch uint64, endEpoch uint64) (map[uint64]map[uint64]bool, error) { + tmr := time.NewTimer(REPORT_TIMEOUT) + defer tmr.Stop() + go func() { + <-tmr.C + logger.WithFields(logrus.Fields{ + "validators": validators, + "startEpoch": startEpoch, + "endEpoch": endEpoch, + }).Errorf("%s call took longer than %v", utils.GetCurrentFuncName(), REPORT_TIMEOUT) + }() if len(validators) == 0 { return nil, fmt.Errorf("passing empty validator array is unsupported") @@ -1229,6 +1347,16 @@ func (bigtable *Bigtable) GetValidatorMissedAttestationHistory(validators []uint } func (bigtable *Bigtable) GetValidatorSyncDutiesHistory(validators []uint64, startSlot uint64, endSlot uint64) (map[uint64]map[uint64]*types.ValidatorSyncParticipation, error) { + tmr := time.NewTimer(REPORT_TIMEOUT) + defer tmr.Stop() + go func() { + <-tmr.C + logger.WithFields(logrus.Fields{ + "validators": validators, + "startSlot": startSlot, + "endSlot": endSlot, + }).Errorf("%s call took longer than %v", utils.GetCurrentFuncName(), REPORT_TIMEOUT) + }() if len(validators) == 0 { return nil, fmt.Errorf("passing empty validator array is unsupported") @@ -1320,6 +1448,17 @@ func (bigtable *Bigtable) GetValidatorSyncDutiesHistory(validators []uint64, sta } func (bigtable *Bigtable) GetValidatorMissedAttestationsCount(validators []uint64, firstEpoch uint64, lastEpoch uint64) (map[uint64]*types.ValidatorMissedAttestationsStatistic, error) { + tmr := time.NewTimer(REPORT_TIMEOUT) + defer tmr.Stop() + go func() { + <-tmr.C + logger.WithFields(logrus.Fields{ + "validators": validators, + "startEpoch": firstEpoch, + "endEpoch": lastEpoch, + }).Errorf("%s call took longer than %v", utils.GetCurrentFuncName(), REPORT_TIMEOUT) + }() + if firstEpoch > lastEpoch { return nil, fmt.Errorf("GetValidatorMissedAttestationsCount received an invalid firstEpoch (%d) and lastEpoch (%d) combination", firstEpoch, lastEpoch) } @@ -1440,6 +1579,16 @@ func (bigtable *Bigtable) GetValidatorEffectiveness(validators []uint64, epoch u } func (bigtable *Bigtable) GetValidatorBalanceStatistics(validators []uint64, startEpoch, endEpoch uint64) (map[uint64]*types.ValidatorBalanceStatistic, error) { + tmr := time.NewTimer(REPORT_TIMEOUT) + defer tmr.Stop() + go func() { + <-tmr.C + logger.WithFields(logrus.Fields{ + "validators": validators, + "startEpoch": startEpoch, + "endEpoch": endEpoch, + }).Errorf("%s call took longer than %v", utils.GetCurrentFuncName(), REPORT_TIMEOUT) + }() type ResultContainer struct { mu sync.Mutex @@ -1516,6 +1665,17 @@ func (bigtable *Bigtable) GetValidatorBalanceStatistics(validators []uint64, sta } func (bigtable *Bigtable) GetValidatorProposalHistory(validators []uint64, startEpoch uint64, endEpoch uint64) (map[uint64][]*types.ValidatorProposal, error) { + tmr := time.NewTimer(REPORT_TIMEOUT) + defer tmr.Stop() + go func() { + <-tmr.C + logger.WithFields(logrus.Fields{ + "validators": validators, + "startEpoch": startEpoch, + "endEpoch": endEpoch, + }).Errorf("%s call took longer than %v", utils.GetCurrentFuncName(), REPORT_TIMEOUT) + }() + if len(validators) == 0 { return nil, fmt.Errorf("passing empty validator array is unsupported") } @@ -1606,6 +1766,7 @@ func (bigtable *Bigtable) GetValidatorProposalHistory(validators []uint64, start } func (bigtable *Bigtable) SaveValidatorIncomeDetails(epoch uint64, rewards map[uint64]*itypes.ValidatorEpochIncome) error { + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) defer cancel() @@ -1689,326 +1850,19 @@ func (bigtable *Bigtable) SaveValidatorIncomeDetails(epoch uint64, rewards map[u return nil } -func (bigtable *Bigtable) MigrateEpochSchemaV1ToV2(epoch uint64) error { - funcStart := time.Now() - - defer func() { - logger.Infof("migration of epoch %v completed in %v", epoch, time.Since(funcStart)) - }() - - // start := time.Now() - - type validatorEpochData struct { - ValidatorIndex uint64 - Proposals map[uint64]uint64 - AttestationTargetSlot uint64 - AttestationInclusionSlot uint64 - SyncParticipation map[uint64]uint64 - EffectiveBalance uint64 - Balance uint64 - IncomeDetails *itypes.ValidatorEpochIncome - } - - epochData := make(map[uint64]*validatorEpochData) - filter := gcp_bigtable.LatestNFilter(1) - ctx := context.Background() - - prefixEpochRange := gcp_bigtable.PrefixRange(fmt.Sprintf("%s:e:b:%s", bigtable.chainId, bigtable.reversedPaddedEpoch(epoch))) - - err := bigtable.tableBeaconchain.ReadRows(ctx, prefixEpochRange, func(r gcp_bigtable.Row) bool { - // logger.Infof("processing row %v", r.Key()) - - keySplit := strings.Split(r.Key(), ":") - - rowKeyEpoch, err := strconv.ParseUint(keySplit[4], 10, 64) - if err != nil { - logger.Errorf("error parsing epoch from row key %v: %v", r.Key(), err) - return false - } - - rowKeyEpoch = MAX_EPOCH - rowKeyEpoch - - if epoch != rowKeyEpoch { - logger.Errorf("retrieved different epoch than requested, requested: %d, retrieved: %d", epoch, rowKeyEpoch) - } - - // logger.Infof("epoch is %d", rowKeyEpoch) - - for columnFamily, readItems := range r { - - for _, ri := range readItems { - - if ri.Column == "stats:sum" { // skip migrating the total epoch income stats - continue - } - - validator, err := strconv.ParseUint(strings.TrimPrefix(ri.Column, columnFamily+":"), 10, 64) - if err != nil { - logger.Errorf("error parsing validator from column key %v: %v", ri.Column, err) - return false - } - - // logger.Infof("retrieved field %s from column family %s for validator %d", ri.Column, columnFamily, validator) - - if epochData[validator] == nil { - epochData[validator] = &validatorEpochData{ - ValidatorIndex: validator, - Proposals: make(map[uint64]uint64), - SyncParticipation: make(map[uint64]uint64), - } - } - - if columnFamily == VALIDATOR_BALANCES_FAMILY { - // logger.Infof("processing balance data for validator %d", validator) - balances := ri.Value - balanceBytes := balances[0:8] - effectiveBalanceBytes := balances[8:16] - epochData[validator].Balance = binary.LittleEndian.Uint64(balanceBytes) - epochData[validator].EffectiveBalance = binary.LittleEndian.Uint64(effectiveBalanceBytes) - } else if columnFamily == INCOME_DETAILS_COLUMN_FAMILY { - // logger.Infof("processing income details data for validator %d", validator) - incomeDetails := &itypes.ValidatorEpochIncome{} - err = proto.Unmarshal(ri.Value, incomeDetails) - if err != nil { - logger.Errorf("error decoding validator income data for row %v: %v", r.Key(), err) - return false - } - - epochData[validator].IncomeDetails = incomeDetails - } else { - logger.Errorf("retrieved unexpected column family %s", columnFamily) - } - } - } - - return true - }, gcp_bigtable.RowFilter(filter)) - - if err != nil { - return err - } - - // logger.Infof("retrieved epoch data for %d validators in %v", len(epochData), time.Since(start)) - // start = time.Now() - - prefixEpochSlotRange := gcp_bigtable.PrefixRange(fmt.Sprintf("%s:e:%s:s:", bigtable.chainId, bigtable.reversedPaddedEpoch(epoch))) - - err = bigtable.tableBeaconchain.ReadRows(ctx, prefixEpochSlotRange, func(r gcp_bigtable.Row) bool { - // logger.Infof("processing row %v", r.Key()) - - keySplit := strings.Split(r.Key(), ":") - - rowKeyEpoch, err := strconv.ParseUint(keySplit[3], 10, 64) - if err != nil { - logger.Errorf("error parsing epoch from row key %v: %v", r.Key(), err) - return false - } - - rowKeyEpoch = MAX_EPOCH - rowKeyEpoch - - if epoch != rowKeyEpoch { - logger.Errorf("retrieved different epoch than requested, requested: %d, retrieved: %d", epoch, rowKeyEpoch) - } - - slot, err := strconv.ParseUint(keySplit[4], 10, 64) - if err != nil { - logger.Errorf("error parsing slot from row key %v: %v", r.Key(), err) - return false - } - slot = MAX_BLOCK_NUMBER - slot - - // logger.Infof("epoch is %d, slot is %d", rowKeyEpoch, slot) - - for columnFamily, readItems := range r { - - for _, ri := range readItems { - - validator, err := strconv.ParseUint(strings.TrimPrefix(ri.Column, columnFamily+":"), 10, 64) - if err != nil { - logger.Errorf("error parsing validator from column key %v: %v", ri.Column, err) - return false - } - - inclusionSlot := uint64(0) - - if ri.Timestamp > 0 { - inclusionSlot = MAX_BLOCK_NUMBER - uint64(ri.Timestamp)/1000 - } - - // logger.Infof("retrieved field %s from column family %s for validator %d", ri.Column, columnFamily, validator) - - if epochData[validator] == nil { - epochData[validator] = &validatorEpochData{ - ValidatorIndex: validator, - Proposals: make(map[uint64]uint64), - SyncParticipation: make(map[uint64]uint64), - } - } - - if columnFamily == ATTESTATIONS_FAMILY { - // logger.Infof("processing balance data for validator %d", validator) - epochData[validator].AttestationTargetSlot = slot - epochData[validator].AttestationInclusionSlot = inclusionSlot - // logger.Infof("processing attestation data for validator %d, target slot %d, inclusion slot %d", validator, slot, inclusionSlot) - } else if columnFamily == PROPOSALS_FAMILY { - epochData[validator].Proposals[slot] = inclusionSlot - // logger.Infof("processing proposer data for validator %d, proposal slot %d, inclusion slot %d", validator, slot, inclusionSlot) - } else if columnFamily == SYNC_COMMITTEES_FAMILY { - epochData[validator].SyncParticipation[slot] = inclusionSlot - //logger.Infof("processing sync data for validator %d, proposal slot %d, inclusion slot %d", validator, slot, inclusionSlot) - } else { - logger.Errorf("retrieved unexpected column family %s", columnFamily) - } - } - } - - return true - }, gcp_bigtable.RowFilter(filter)) - - if err != nil { - return err - } - - // logger.Infof("retrieved slot data for %d validators in %v", len(epochData), time.Since(start)) - // start = time.Now() - - // save validator balance data - validators := make([]*types.Validator, 0, len(epochData)) - - for _, validator := range epochData { - validators = append(validators, &types.Validator{ - Index: validator.ValidatorIndex, - EffectiveBalance: validator.EffectiveBalance, - Balance: validator.Balance, - }) - } - - err = bigtable.SaveValidatorBalances(epoch, validators) - if err != nil { - return err - } - // logger.Infof("migrated balance data in %v", time.Since(start)) - // start = time.Now() - - mutsInclusionSlot := make([]*gcp_bigtable.Mutation, 0, MAX_BATCH_MUTATIONS) - keysInclusionSlot := make([]string, 0, MAX_BATCH_MUTATIONS) - - for _, validator := range epochData { - mutInclusionSlot := gcp_bigtable.NewMutation() - mutInclusionSlot.Set(ATTESTATIONS_FAMILY, fmt.Sprintf("%d", validator.AttestationTargetSlot), gcp_bigtable.Timestamp((MAX_BLOCK_NUMBER-validator.AttestationInclusionSlot)*1000), []byte{}) - key := fmt.Sprintf("%s:%s:%s:%s", bigtable.chainId, ATTESTATIONS_FAMILY, bigtable.validatorIndexToKey(validator.ValidatorIndex), bigtable.reversedPaddedEpoch(epoch)) - - mutsInclusionSlot = append(mutsInclusionSlot, mutInclusionSlot) - keysInclusionSlot = append(keysInclusionSlot, key) - - if len(mutsInclusionSlot) == MAX_BATCH_MUTATIONS { - errs, err := bigtable.tableValidatorsHistory.ApplyBulk(ctx, keysInclusionSlot, mutsInclusionSlot) - if err != nil { - return err - } - for _, err := range errs { - return err - } - mutsInclusionSlot = make([]*gcp_bigtable.Mutation, 0, MAX_BATCH_MUTATIONS) - keysInclusionSlot = make([]string, 0, MAX_BATCH_MUTATIONS) - } - } - - if len(mutsInclusionSlot) > 0 { - errs, err := bigtable.tableValidatorsHistory.ApplyBulk(ctx, keysInclusionSlot, mutsInclusionSlot) - - if err != nil { - return err - } - - for _, err := range errs { - return err - } - } - // logger.Infof("migrated attestation data in %v", time.Since(start)) - // start = time.Now() - - mutsProposals := make([]*gcp_bigtable.Mutation, 0, MAX_BATCH_MUTATIONS) - keysProposals := make([]string, 0, MAX_BATCH_MUTATIONS) - - for _, validator := range epochData { - if len(validator.Proposals) == 0 { - continue - } - for slot, inclusionSlot := range validator.Proposals { - mut := gcp_bigtable.NewMutation() - mut.Set(PROPOSALS_FAMILY, "b", gcp_bigtable.Timestamp((MAX_BLOCK_NUMBER-inclusionSlot)*1000), []byte{}) - key := fmt.Sprintf("%s:%s:%s:%s:%s", bigtable.chainId, bigtable.validatorIndexToKey(validator.ValidatorIndex), PROPOSALS_FAMILY, bigtable.reversedPaddedEpoch(epoch), bigtable.reversedPaddedSlot(slot)) - - mutsProposals = append(mutsProposals, mut) - keysProposals = append(keysProposals, key) - } - } - errs, err := bigtable.tableValidatorsHistory.ApplyBulk(ctx, keysProposals, mutsProposals) - - if err != nil { - return err - } - - for _, err := range errs { - return err - } - // logger.Infof("migrated proposal data in %v", time.Since(start)) - // start = time.Now() - - mutsSync := make([]*gcp_bigtable.Mutation, 0, MAX_BATCH_MUTATIONS) - keysSync := make([]string, 0, MAX_BATCH_MUTATIONS) - - for _, validator := range epochData { - - if len(validator.SyncParticipation) == 0 { - continue - } - for slot, inclusionSlot := range validator.SyncParticipation { - mut := gcp_bigtable.NewMutation() - mut.Set(SYNC_COMMITTEES_FAMILY, "s", gcp_bigtable.Timestamp((MAX_BLOCK_NUMBER-inclusionSlot)*1000), []byte{}) - - key := fmt.Sprintf("%s:%s:%s:%s:%s", bigtable.chainId, bigtable.validatorIndexToKey(validator.ValidatorIndex), SYNC_COMMITTEES_FAMILY, bigtable.reversedPaddedEpoch(epoch), bigtable.reversedPaddedSlot(slot)) - mutsSync = append(mutsSync, mut) - keysSync = append(keysSync, key) - } - } - - errs, err = bigtable.tableValidatorsHistory.ApplyBulk(ctx, keysSync, mutsSync) - - if err != nil { - return err - } - - for _, err := range errs { - return err - } - // logger.Infof("migrated sync data in %v", time.Since(start)) - // start = time.Now() - - incomeData := make(map[uint64]*itypes.ValidatorEpochIncome) - for _, validator := range epochData { - if validator.IncomeDetails == nil { - continue - } - incomeData[validator.ValidatorIndex] = validator.IncomeDetails - } - - err = bigtable.SaveValidatorIncomeDetails(epoch, incomeData) - if err != nil { - return err - } - - // logger.Infof("migrated income data in %v", time.Since(start)) - // start = time.Now() - - return nil - -} - // GetValidatorIncomeDetailsHistory returns the validator income details // startEpoch & endEpoch are inclusive func (bigtable *Bigtable) GetValidatorIncomeDetailsHistory(validators []uint64, startEpoch uint64, endEpoch uint64) (map[uint64]map[uint64]*itypes.ValidatorEpochIncome, error) { + tmr := time.NewTimer(REPORT_TIMEOUT) + defer tmr.Stop() + go func() { + <-tmr.C + logger.WithFields(logrus.Fields{ + "validators": validators, + "startEpoch": startEpoch, + "endEpoch": endEpoch, + }).Errorf("%s call took longer than %v", utils.GetCurrentFuncName(), REPORT_TIMEOUT) + }() if len(validators) == 0 { return nil, fmt.Errorf("passing empty validator array is unsupported") diff --git a/db/bigtable_eth1.go b/db/bigtable_eth1.go index 19a24102d0..e93b3cedef 100644 --- a/db/bigtable_eth1.go +++ b/db/bigtable_eth1.go @@ -134,6 +134,14 @@ func (bigtable *Bigtable) SaveBlock(block *types.Eth1Block) error { } func (bigtable *Bigtable) GetBlockFromBlocksTable(number uint64) (*types.Eth1Block, error) { + tmr := time.NewTimer(REPORT_TIMEOUT) + defer tmr.Stop() + go func() { + <-tmr.C + logger.WithFields(logrus.Fields{ + "validators": number, + }).Errorf("%s call took longer than %v", utils.GetCurrentFuncName(), REPORT_TIMEOUT) + }() ctx, cancel := context.WithTimeout(context.Background(), time.Second*30) defer cancel() @@ -200,6 +208,13 @@ func (bigtable *Bigtable) CheckForGapsInBlocksTable(lookback int) (gapFound bool } func (bigtable *Bigtable) GetLastBlockInBlocksTable() (int, error) { + tmr := time.NewTimer(REPORT_TIMEOUT) + defer tmr.Stop() + go func() { + <-tmr.C + logger.Errorf("%s call took longer than %v", utils.GetCurrentFuncName(), REPORT_TIMEOUT) + }() + ctx, cancel := context.WithTimeout(context.Background(), time.Second*30) defer cancel() redisKey := bigtable.chainId + ":lastBlockInBlocksTable" @@ -275,6 +290,13 @@ func (bigtable *Bigtable) CheckForGapsInDataTable(lookback int) error { } func (bigtable *Bigtable) GetLastBlockInDataTable() (int, error) { + tmr := time.NewTimer(REPORT_TIMEOUT) + defer tmr.Stop() + go func() { + <-tmr.C + logger.Errorf("%s call took longer than %v", utils.GetCurrentFuncName(), REPORT_TIMEOUT) + }() + ctx, cancel := context.WithTimeout(context.Background(), time.Second*30) defer cancel() redisKey := bigtable.chainId + ":lastBlockInDataTable" @@ -365,6 +387,13 @@ func (bigtable *Bigtable) SetLastBlockInDataTable(lastBlock int64) error { } func (bigtable *Bigtable) GetMostRecentBlockFromDataTable() (*types.Eth1BlockIndexed, error) { + tmr := time.NewTimer(REPORT_TIMEOUT) + defer tmr.Stop() + go func() { + <-tmr.C + logger.Errorf("%s call took longer than %v", utils.GetCurrentFuncName(), REPORT_TIMEOUT) + }() + ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(time.Second*30)) defer cancel() @@ -424,6 +453,16 @@ func getBlockHandler(blocks *[]*types.Eth1BlockIndexed) func(gcp_bigtable.Row) b // high: highest (max) block number // low: lowest (min) block number func (bigtable *Bigtable) GetFullBlocksDescending(stream chan<- *types.Eth1Block, high, low uint64) error { + tmr := time.NewTimer(REPORT_TIMEOUT) + defer tmr.Stop() + go func() { + <-tmr.C + logger.WithFields(logrus.Fields{ + "high": high, + "low": low, + }).Errorf("%s call took longer than %v", utils.GetCurrentFuncName(), REPORT_TIMEOUT) + }() + ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(time.Second*180)) defer cancel() @@ -475,6 +514,16 @@ func (bigtable *Bigtable) GetFullBlocksDescending(stream chan<- *types.Eth1Block } func (bigtable *Bigtable) GetBlocksIndexedMultiple(blockNumbers []uint64, limit uint64) ([]*types.Eth1BlockIndexed, error) { + tmr := time.NewTimer(REPORT_TIMEOUT) + defer tmr.Stop() + go func() { + <-tmr.C + logger.WithFields(logrus.Fields{ + "blockNumbers": blockNumbers, + "limit": limit, + }).Errorf("%s call took longer than %v", utils.GetCurrentFuncName(), REPORT_TIMEOUT) + }() + rowList := gcp_bigtable.RowList{} for _, block := range blockNumbers { rowList = append(rowList, fmt.Sprintf("%s:B:%s", bigtable.chainId, reversedPaddedBlockNumber(block))) @@ -501,6 +550,16 @@ func (bigtable *Bigtable) GetBlocksIndexedMultiple(blockNumbers []uint64, limit // GetBlocksDescending gets blocks starting at block start func (bigtable *Bigtable) GetBlocksDescending(start, limit uint64) ([]*types.Eth1BlockIndexed, error) { + tmr := time.NewTimer(REPORT_TIMEOUT) + defer tmr.Stop() + go func() { + <-tmr.C + logger.WithFields(logrus.Fields{ + "start": start, + "limit": limit, + }).Errorf("%s call took longer than %v", utils.GetCurrentFuncName(), REPORT_TIMEOUT) + }() + if start < 1 || limit < 1 || limit > start { return nil, fmt.Errorf("invalid block range provided (start: %v, limit: %v)", start, limit) } @@ -1814,6 +1873,16 @@ func (bigtable *Bigtable) TransformWithdrawals(block *types.Eth1Block, cache *fr } func (bigtable *Bigtable) GetEth1TxForAddress(prefix string, limit int64) ([]*types.Eth1TransactionIndexed, string, error) { + tmr := time.NewTimer(REPORT_TIMEOUT) + defer tmr.Stop() + go func() { + <-tmr.C + logger.WithFields(logrus.Fields{ + "prefix": prefix, + "limit": limit, + }).Errorf("%s call took longer than %v", utils.GetCurrentFuncName(), REPORT_TIMEOUT) + }() + ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(time.Second*30)) defer cancel() @@ -1863,6 +1932,16 @@ func (bigtable *Bigtable) GetEth1TxForAddress(prefix string, limit int64) ([]*ty } func (bigtable *Bigtable) GetAddressesNamesArMetadata(names *map[string]string, inputMetadata *map[string]*types.ERC20Metadata) (map[string]string, map[string]*types.ERC20Metadata, error) { + tmr := time.NewTimer(REPORT_TIMEOUT) + defer tmr.Stop() + go func() { + <-tmr.C + logger.WithFields(logrus.Fields{ + "names": names, + "inputMetadata": inputMetadata, + }).Errorf("%s call took longer than %v", utils.GetCurrentFuncName(), REPORT_TIMEOUT) + }() + outputMetadata := make(map[string]*types.ERC20Metadata) g := new(errgroup.Group) @@ -1904,6 +1983,15 @@ func (bigtable *Bigtable) GetAddressesNamesArMetadata(names *map[string]string, } func (bigtable *Bigtable) GetIndexedEth1Transaction(txHash []byte) (*types.Eth1TransactionIndexed, error) { + tmr := time.NewTimer(REPORT_TIMEOUT) + defer tmr.Stop() + go func() { + <-tmr.C + logger.WithFields(logrus.Fields{ + "txHash": txHash, + }).Errorf("%s call took longer than %v", utils.GetCurrentFuncName(), REPORT_TIMEOUT) + }() + ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(time.Second*30)) defer cancel() key := fmt.Sprintf("%s:TX:%x", bigtable.chainId, txHash) @@ -1926,6 +2014,17 @@ func (bigtable *Bigtable) GetIndexedEth1Transaction(txHash []byte) (*types.Eth1T } func (bigtable *Bigtable) GetAddressTransactionsTableData(address []byte, search string, pageToken string) (*types.DataTableResponse, error) { + tmr := time.NewTimer(REPORT_TIMEOUT) + defer tmr.Stop() + go func() { + <-tmr.C + logger.WithFields(logrus.Fields{ + "address": address, + "search": search, + "pageToken": pageToken, + }).Errorf("%s call took longer than %v", utils.GetCurrentFuncName(), REPORT_TIMEOUT) + }() + if pageToken == "" { pageToken = fmt.Sprintf("%s:I:TX:%x:%s:", bigtable.chainId, address, FILTER_TIME) } @@ -1977,6 +2076,16 @@ func (bigtable *Bigtable) GetAddressTransactionsTableData(address []byte, search } func (bigtable *Bigtable) GetEth1BlocksForAddress(prefix string, limit int64) ([]*types.Eth1BlockIndexed, string, error) { + tmr := time.NewTimer(REPORT_TIMEOUT) + defer tmr.Stop() + go func() { + <-tmr.C + logger.WithFields(logrus.Fields{ + "prefix": prefix, + "limit": limit, + }).Errorf("%s call took longer than %v", utils.GetCurrentFuncName(), REPORT_TIMEOUT) + }() + ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(time.Second*30)) defer cancel() @@ -2026,6 +2135,17 @@ func (bigtable *Bigtable) GetEth1BlocksForAddress(prefix string, limit int64) ([ } func (bigtable *Bigtable) GetAddressBlocksMinedTableData(address string, search string, pageToken string) (*types.DataTableResponse, error) { + tmr := time.NewTimer(REPORT_TIMEOUT) + defer tmr.Stop() + go func() { + <-tmr.C + logger.WithFields(logrus.Fields{ + "address": address, + "search": search, + "pageToken": pageToken, + }).Errorf("%s call took longer than %v", utils.GetCurrentFuncName(), REPORT_TIMEOUT) + }() + if pageToken == "" { pageToken = fmt.Sprintf("%s:I:B:%s:", bigtable.chainId, address) } @@ -2056,6 +2176,16 @@ func (bigtable *Bigtable) GetAddressBlocksMinedTableData(address string, search } func (bigtable *Bigtable) GetEth1UnclesForAddress(prefix string, limit int64) ([]*types.Eth1UncleIndexed, string, error) { + tmr := time.NewTimer(REPORT_TIMEOUT) + defer tmr.Stop() + go func() { + <-tmr.C + logger.WithFields(logrus.Fields{ + "prefix": prefix, + "limit": limit, + }).Errorf("%s call took longer than %v", utils.GetCurrentFuncName(), REPORT_TIMEOUT) + }() + ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(time.Second*30)) defer cancel() @@ -2105,6 +2235,17 @@ func (bigtable *Bigtable) GetEth1UnclesForAddress(prefix string, limit int64) ([ } func (bigtable *Bigtable) GetAddressUnclesMinedTableData(address string, search string, pageToken string) (*types.DataTableResponse, error) { + tmr := time.NewTimer(REPORT_TIMEOUT) + defer tmr.Stop() + go func() { + <-tmr.C + logger.WithFields(logrus.Fields{ + "address": address, + "search": search, + "pageToken": pageToken, + }).Errorf("%s call took longer than %v", utils.GetCurrentFuncName(), REPORT_TIMEOUT) + }() + if pageToken == "" { pageToken = fmt.Sprintf("%s:I:U:%s:", bigtable.chainId, address) } @@ -2133,6 +2274,16 @@ func (bigtable *Bigtable) GetAddressUnclesMinedTableData(address string, search } func (bigtable *Bigtable) GetEth1ItxForAddress(prefix string, limit int64) ([]*types.Eth1InternalTransactionIndexed, string, error) { + tmr := time.NewTimer(REPORT_TIMEOUT) + defer tmr.Stop() + go func() { + <-tmr.C + logger.WithFields(logrus.Fields{ + "prefix": prefix, + "limit": limit, + }).Errorf("%s call took longer than %v", utils.GetCurrentFuncName(), REPORT_TIMEOUT) + }() + ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(time.Second*30)) defer cancel() @@ -2186,6 +2337,17 @@ func (bigtable *Bigtable) GetEth1ItxForAddress(prefix string, limit int64) ([]*t } func (bigtable *Bigtable) GetAddressInternalTableData(address []byte, search string, pageToken string) (*types.DataTableResponse, error) { + tmr := time.NewTimer(REPORT_TIMEOUT) + defer tmr.Stop() + go func() { + <-tmr.C + logger.WithFields(logrus.Fields{ + "address": address, + "search": search, + "pageToken": pageToken, + }).Errorf("%s call took longer than %v", utils.GetCurrentFuncName(), REPORT_TIMEOUT) + }() + // defaults to most recent if pageToken == "" { pageToken = fmt.Sprintf("%s:I:ITX:%x:%s:", bigtable.chainId, address, FILTER_TIME) @@ -2235,6 +2397,16 @@ func (bigtable *Bigtable) GetAddressInternalTableData(address []byte, search str } func (bigtable *Bigtable) GetInternalTransfersForTransaction(transaction []byte, from []byte) ([]types.Transfer, error) { + tmr := time.NewTimer(REPORT_TIMEOUT) + defer tmr.Stop() + go func() { + <-tmr.C + logger.WithFields(logrus.Fields{ + "transaction": transaction, + "from": from, + }).Errorf("%s call took longer than %v", utils.GetCurrentFuncName(), REPORT_TIMEOUT) + }() + ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(time.Second*30)) defer cancel() @@ -2311,6 +2483,15 @@ func (bigtable *Bigtable) GetInternalTransfersForTransaction(transaction []byte, // currently only erc20 func (bigtable *Bigtable) GetArbitraryTokenTransfersForTransaction(transaction []byte) ([]*types.Transfer, error) { + tmr := time.NewTimer(REPORT_TIMEOUT) + defer tmr.Stop() + go func() { + <-tmr.C + logger.WithFields(logrus.Fields{ + "transaction": transaction, + }).Errorf("%s call took longer than %v", utils.GetCurrentFuncName(), REPORT_TIMEOUT) + }() + ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(time.Second*30)) defer cancel() // uses a more standard transfer in-between type so multiple token types can be handle before the final table response is generated @@ -2420,6 +2601,16 @@ func (bigtable *Bigtable) GetArbitraryTokenTransfersForTransaction(transaction [ } func (bigtable *Bigtable) GetEth1ERC20ForAddress(prefix string, limit int64) ([]*types.Eth1ERC20Indexed, string, error) { + tmr := time.NewTimer(REPORT_TIMEOUT) + defer tmr.Stop() + go func() { + <-tmr.C + logger.WithFields(logrus.Fields{ + "prefix": prefix, + "limit": limit, + }).Errorf("%s call took longer than %v", utils.GetCurrentFuncName(), REPORT_TIMEOUT) + }() + ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(time.Second*30)) defer cancel() @@ -2467,6 +2658,16 @@ func (bigtable *Bigtable) GetEth1ERC20ForAddress(prefix string, limit int64) ([] } func (bigtable *Bigtable) GetAddressErc20TableData(address []byte, search string, pageToken string) (*types.DataTableResponse, error) { + tmr := time.NewTimer(REPORT_TIMEOUT) + defer tmr.Stop() + go func() { + <-tmr.C + logger.WithFields(logrus.Fields{ + "address": address, + "search": search, + "pageToken": pageToken, + }).Errorf("%s call took longer than %v", utils.GetCurrentFuncName(), REPORT_TIMEOUT) + }() if pageToken == "" { pageToken = fmt.Sprintf("%s:I:ERC20:%x:%s:", bigtable.chainId, address, FILTER_TIME) @@ -2526,6 +2727,16 @@ func (bigtable *Bigtable) GetAddressErc20TableData(address []byte, search string } func (bigtable *Bigtable) GetEth1ERC721ForAddress(prefix string, limit int64) ([]*types.Eth1ERC721Indexed, string, error) { + tmr := time.NewTimer(REPORT_TIMEOUT) + defer tmr.Stop() + go func() { + <-tmr.C + logger.WithFields(logrus.Fields{ + "prefix": prefix, + "limit": limit, + }).Errorf("%s call took longer than %v", utils.GetCurrentFuncName(), REPORT_TIMEOUT) + }() + ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(time.Second*30)) defer cancel() @@ -2577,6 +2788,16 @@ func (bigtable *Bigtable) GetEth1ERC721ForAddress(prefix string, limit int64) ([ } func (bigtable *Bigtable) GetAddressErc721TableData(address string, search string, pageToken string) (*types.DataTableResponse, error) { + tmr := time.NewTimer(REPORT_TIMEOUT) + defer tmr.Stop() + go func() { + <-tmr.C + logger.WithFields(logrus.Fields{ + "address": address, + "search": search, + "pageToken": pageToken, + }).Errorf("%s call took longer than %v", utils.GetCurrentFuncName(), REPORT_TIMEOUT) + }() if pageToken == "" { pageToken = fmt.Sprintf("%s:I:ERC721:%s:%s:", bigtable.chainId, address, FILTER_TIME) @@ -2624,6 +2845,16 @@ func (bigtable *Bigtable) GetAddressErc721TableData(address string, search strin } func (bigtable *Bigtable) GetEth1ERC1155ForAddress(prefix string, limit int64) ([]*types.ETh1ERC1155Indexed, string, error) { + tmr := time.NewTimer(REPORT_TIMEOUT) + defer tmr.Stop() + go func() { + <-tmr.C + logger.WithFields(logrus.Fields{ + "prefix": prefix, + "limit": limit, + }).Errorf("%s call took longer than %v", utils.GetCurrentFuncName(), REPORT_TIMEOUT) + }() + ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(time.Second*30)) defer cancel() @@ -2672,6 +2903,17 @@ func (bigtable *Bigtable) GetEth1ERC1155ForAddress(prefix string, limit int64) ( } func (bigtable *Bigtable) GetAddressErc1155TableData(address string, search string, pageToken string) (*types.DataTableResponse, error) { + tmr := time.NewTimer(REPORT_TIMEOUT) + defer tmr.Stop() + go func() { + <-tmr.C + logger.WithFields(logrus.Fields{ + "address": address, + "search": search, + "pageToken": pageToken, + }).Errorf("%s call took longer than %v", utils.GetCurrentFuncName(), REPORT_TIMEOUT) + }() + if pageToken == "" { pageToken = fmt.Sprintf("%s:I:ERC1155:%s:%s:", bigtable.chainId, address, FILTER_TIME) } @@ -2719,6 +2961,16 @@ func (bigtable *Bigtable) GetAddressErc1155TableData(address string, search stri } func (bigtable *Bigtable) GetMetadataUpdates(prefix string, startToken string, limit int) ([]string, []*types.Eth1AddressBalance, error) { + tmr := time.NewTimer(REPORT_TIMEOUT) + defer tmr.Stop() + go func() { + <-tmr.C + logger.WithFields(logrus.Fields{ + "prefix": prefix, + "startToken": startToken, + }).Errorf("%s call took longer than %v", utils.GetCurrentFuncName(), REPORT_TIMEOUT) + }() + ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(time.Minute*120)) defer cancel() @@ -2746,6 +2998,16 @@ func (bigtable *Bigtable) GetMetadataUpdates(prefix string, startToken string, l } func (bigtable *Bigtable) GetMetadata(startToken string, limit int) ([]string, []*types.Eth1AddressBalance, error) { + tmr := time.NewTimer(REPORT_TIMEOUT) + defer tmr.Stop() + go func() { + <-tmr.C + logger.WithFields(logrus.Fields{ + "startToken": startToken, + "limit": limit, + }).Errorf("%s call took longer than %v", utils.GetCurrentFuncName(), REPORT_TIMEOUT) + }() + ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(time.Minute*120)) defer cancel() @@ -2775,6 +3037,15 @@ func (bigtable *Bigtable) GetMetadata(startToken string, limit int) ([]string, [ } func (bigtable *Bigtable) GetMetadataForAddress(address []byte) (*types.Eth1AddressMetadata, error) { + tmr := time.NewTimer(REPORT_TIMEOUT) + defer tmr.Stop() + go func() { + <-tmr.C + logger.WithFields(logrus.Fields{ + "address": address, + }).Errorf("%s call took longer than %v", utils.GetCurrentFuncName(), REPORT_TIMEOUT) + }() + ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(time.Second*30)) defer cancel() @@ -2876,6 +3147,16 @@ func (bigtable *Bigtable) GetMetadataForAddress(address []byte) (*types.Eth1Addr } func (bigtable *Bigtable) GetBalanceForAddress(address []byte, token []byte) (*types.Eth1AddressBalance, error) { + tmr := time.NewTimer(REPORT_TIMEOUT) + defer tmr.Stop() + go func() { + <-tmr.C + logger.WithFields(logrus.Fields{ + "address": address, + "token": token, + }).Errorf("%s call took longer than %v", utils.GetCurrentFuncName(), REPORT_TIMEOUT) + }() + ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(time.Second*30)) defer cancel() @@ -2913,6 +3194,14 @@ func (bigtable *Bigtable) GetBalanceForAddress(address []byte, token []byte) (*t } func (bigtable *Bigtable) GetERC20MetadataForAddress(address []byte) (*types.ERC20Metadata, error) { + tmr := time.NewTimer(REPORT_TIMEOUT) + defer tmr.Stop() + go func() { + <-tmr.C + logger.WithFields(logrus.Fields{ + "address": address, + }).Errorf("%s call took longer than %v", utils.GetCurrentFuncName(), REPORT_TIMEOUT) + }() if len(address) == 1 { return &types.ERC20Metadata{ @@ -3041,6 +3330,15 @@ func (bigtable *Bigtable) SaveERC20Metadata(address []byte, metadata *types.ERC2 } func (bigtable *Bigtable) GetAddressName(address []byte) (string, error) { + tmr := time.NewTimer(REPORT_TIMEOUT) + defer tmr.Stop() + go func() { + <-tmr.C + logger.WithFields(logrus.Fields{ + "address": address, + }).Errorf("%s call took longer than %v", utils.GetCurrentFuncName(), REPORT_TIMEOUT) + }() + ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(time.Second*30)) defer cancel() @@ -3074,6 +3372,15 @@ func (bigtable *Bigtable) GetAddressName(address []byte) (string, error) { } func (bigtable *Bigtable) GetAddressNames(addresses map[string]string) error { + tmr := time.NewTimer(REPORT_TIMEOUT) + defer tmr.Stop() + go func() { + <-tmr.C + logger.WithFields(logrus.Fields{ + "addresses": addresses, + }).Errorf("%s call took longer than %v", utils.GetCurrentFuncName(), REPORT_TIMEOUT) + }() + if len(addresses) == 0 { return nil } @@ -3117,6 +3424,15 @@ func (bigtable *Bigtable) SaveAddressName(address []byte, name string) error { } func (bigtable *Bigtable) GetContractMetadata(address []byte) (*types.ContractMetadata, error) { + tmr := time.NewTimer(REPORT_TIMEOUT) + defer tmr.Stop() + go func() { + <-tmr.C + logger.WithFields(logrus.Fields{ + "address": address, + }).Errorf("%s call took longer than %v", utils.GetCurrentFuncName(), REPORT_TIMEOUT) + }() + ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(time.Second*30)) defer cancel() @@ -3296,6 +3612,16 @@ func (bigtable *Bigtable) SaveBlockKeys(blockNumber uint64, blockHash []byte, ke } func (bigtable *Bigtable) GetBlockKeys(blockNumber uint64, blockHash []byte) ([]string, error) { + tmr := time.NewTimer(REPORT_TIMEOUT) + defer tmr.Stop() + go func() { + <-tmr.C + logger.WithFields(logrus.Fields{ + "blockNumber": blockNumber, + "blockHash": blockHash, + }).Errorf("%s call took longer than %v", utils.GetCurrentFuncName(), REPORT_TIMEOUT) + }() + ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(time.Second*30)) defer cancel() @@ -3357,6 +3683,16 @@ func (bigtable *Bigtable) DeleteBlock(blockNumber uint64, blockHash []byte) erro } func (bigtable *Bigtable) GetEth1TxForToken(prefix string, limit int64) ([]*types.Eth1ERC20Indexed, string, error) { + tmr := time.NewTimer(REPORT_TIMEOUT) + defer tmr.Stop() + go func() { + <-tmr.C + logger.WithFields(logrus.Fields{ + "prefix": prefix, + "limit": limit, + }).Errorf("%s call took longer than %v", utils.GetCurrentFuncName(), REPORT_TIMEOUT) + }() + ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(time.Second*30)) defer cancel() @@ -3467,6 +3803,16 @@ func (bigtable *Bigtable) GetTokenTransactionsTableData(token []byte, address [] } func (bigtable *Bigtable) SearchForAddress(addressPrefix []byte, limit int) ([]*types.Eth1AddressSearchItem, error) { + tmr := time.NewTimer(REPORT_TIMEOUT) + defer tmr.Stop() + go func() { + <-tmr.C + logger.WithFields(logrus.Fields{ + "addressPrefix": addressPrefix, + "limit": limit, + }).Errorf("%s call took longer than %v", utils.GetCurrentFuncName(), REPORT_TIMEOUT) + }() + ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(time.Second*30)) defer cancel() @@ -3511,6 +3857,15 @@ func getSignaturePrefix(st types.SignatureType) string { // Get the status of the last signature import run func (bigtable *Bigtable) GetSignatureImportStatus(st types.SignatureType) (*types.SignatureImportStatus, error) { + tmr := time.NewTimer(REPORT_TIMEOUT) + defer tmr.Stop() + go func() { + <-tmr.C + logger.WithFields(logrus.Fields{ + "st": st, + }).Errorf("%s call took longer than %v", utils.GetCurrentFuncName(), REPORT_TIMEOUT) + }() + ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(time.Second*30)) defer cancel() key := fmt.Sprintf("1:%v_SIGNATURE_IMPORT_STATUS", getSignaturePrefix(st)) @@ -3592,6 +3947,16 @@ func (bigtable *Bigtable) SaveSignatures(signatures []types.Signature, st types. // get a signature by it's hex representation func (bigtable *Bigtable) GetSignature(hex string, st types.SignatureType) (*string, error) { + tmr := time.NewTimer(REPORT_TIMEOUT) + defer tmr.Stop() + go func() { + <-tmr.C + logger.WithFields(logrus.Fields{ + "hex": hex, + "st": st, + }).Errorf("%s call took longer than %v", utils.GetCurrentFuncName(), REPORT_TIMEOUT) + }() + ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(time.Second*30)) defer cancel() key := fmt.Sprintf("1:%v_SIGNATURE:%v", getSignaturePrefix(st), hex) @@ -3713,6 +4078,16 @@ func (bigtable *Bigtable) SaveGasNowHistory(slow, standard, rapid, fast *big.Int } func (bigtable *Bigtable) GetGasNowHistory(ts, pastTs time.Time) ([]types.GasNowHistory, error) { + tmr := time.NewTimer(REPORT_TIMEOUT) + defer tmr.Stop() + go func() { + <-tmr.C + logger.WithFields(logrus.Fields{ + "ts": ts, + "pastTs": pastTs, + }).Errorf("%s call took longer than %v", utils.GetCurrentFuncName(), REPORT_TIMEOUT) + }() + ctx, done := context.WithTimeout(context.Background(), time.Second*30) defer done() diff --git a/handlers/validator.go b/handlers/validator.go index 94270affb3..b4e026ec2e 100644 --- a/handlers/validator.go +++ b/handlers/validator.go @@ -20,6 +20,7 @@ import ( "strings" "time" + "github.com/davecgh/go-spew/spew" "github.com/ethereum/go-ethereum/accounts" "github.com/ethereum/go-ethereum/crypto" "github.com/lib/pq" @@ -365,7 +366,6 @@ func Validator(w http.ResponseWriter, r *http.Request) { if lastStatsDay > 30 { lowerBoundDay = lastStatsDay - 30 } - g := errgroup.Group{} g.Go(func() error { start := time.Now() @@ -702,7 +702,8 @@ func Validator(w http.ResponseWriter, r *http.Request) { } lastSyncPeriod := actualSyncPeriods[0] if lastSyncPeriod.LastEpoch > lastExportedEpoch { - res, err := db.BigtableClient.GetValidatorSyncDutiesHistory([]uint64{index}, lastExportedEpoch+1*utils.Config.Chain.Config.SlotsPerEpoch, latestProposedSlot) + spew.Dump(lastExportedEpoch, []uint64{index}, (lastExportedEpoch+1)*utils.Config.Chain.Config.SlotsPerEpoch, latestProposedSlot) + res, err := db.BigtableClient.GetValidatorSyncDutiesHistory([]uint64{index}, (lastExportedEpoch+1)*utils.Config.Chain.Config.SlotsPerEpoch, latestProposedSlot) if err != nil { return fmt.Errorf("error retrieving validator sync participations data from bigtable: %v", err) } diff --git a/utils/utils.go b/utils/utils.go index 6ec467658a..4784f3c776 100644 --- a/utils/utils.go +++ b/utils/utils.go @@ -1538,3 +1538,8 @@ func ReverseString(s string) string { } return string(runes) } + +func GetCurrentFuncName() string { + pc, _, _, _ := runtime.Caller(1) + return fmt.Sprintf("%s", runtime.FuncForPC(pc).Name()) +} From 2f23ca7a46340869fcfc07bbed6eda37881b3944 Mon Sep 17 00:00:00 2001 From: peter <1674920+peterbitfly@users.noreply.github.com> Date: Mon, 11 Sep 2023 10:34:59 +0200 Subject: [PATCH 092/106] (BIDS-2379) remove debug logging --- handlers/validator.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/handlers/validator.go b/handlers/validator.go index b4e026ec2e..e65c5bd4da 100644 --- a/handlers/validator.go +++ b/handlers/validator.go @@ -20,7 +20,6 @@ import ( "strings" "time" - "github.com/davecgh/go-spew/spew" "github.com/ethereum/go-ethereum/accounts" "github.com/ethereum/go-ethereum/crypto" "github.com/lib/pq" @@ -702,7 +701,6 @@ func Validator(w http.ResponseWriter, r *http.Request) { } lastSyncPeriod := actualSyncPeriods[0] if lastSyncPeriod.LastEpoch > lastExportedEpoch { - spew.Dump(lastExportedEpoch, []uint64{index}, (lastExportedEpoch+1)*utils.Config.Chain.Config.SlotsPerEpoch, latestProposedSlot) res, err := db.BigtableClient.GetValidatorSyncDutiesHistory([]uint64{index}, (lastExportedEpoch+1)*utils.Config.Chain.Config.SlotsPerEpoch, latestProposedSlot) if err != nil { return fmt.Errorf("error retrieving validator sync participations data from bigtable: %v", err) From 87af685913b6a62fbcca0f521a075a4a00224051 Mon Sep 17 00:00:00 2001 From: peter <1674920+peterbitfly@users.noreply.github.com> Date: Mon, 11 Sep 2023 10:40:06 +0200 Subject: [PATCH 093/106] (BIDS-2379) fix linter --- utils/utils.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utils/utils.go b/utils/utils.go index 4784f3c776..341339bdda 100644 --- a/utils/utils.go +++ b/utils/utils.go @@ -1541,5 +1541,5 @@ func ReverseString(s string) string { func GetCurrentFuncName() string { pc, _, _, _ := runtime.Caller(1) - return fmt.Sprintf("%s", runtime.FuncForPC(pc).Name()) + return runtime.FuncForPC(pc).Name() } From a2b491e8836000b64c8c1ad7d98a0a54839d54bf Mon Sep 17 00:00:00 2001 From: peter <1674920+peterbitfly@users.noreply.github.com> Date: Mon, 11 Sep 2023 11:31:52 +0200 Subject: [PATCH 094/106] (BIDS-2379) add new bigtable schema migration tool --- cmd/migrations/bigtable/main.go | 94 ++------------------------------- exporter/exporter.go | 7 --- 2 files changed, 3 insertions(+), 98 deletions(-) diff --git a/cmd/migrations/bigtable/main.go b/cmd/migrations/bigtable/main.go index 6c5a7696fe..60359f3e58 100644 --- a/cmd/migrations/bigtable/main.go +++ b/cmd/migrations/bigtable/main.go @@ -2,6 +2,7 @@ package main import ( "eth2-exporter/db" + "eth2-exporter/exporter" "eth2-exporter/rpc" "eth2-exporter/types" "eth2-exporter/utils" @@ -12,7 +13,6 @@ import ( "time" "github.com/sirupsen/logrus" - "golang.org/x/sync/errgroup" ) func main() { @@ -58,60 +58,12 @@ func main() { i := i logrus.Infof("exporting epoch %v", i) - - logrus.Infof("deleting existing epoch data") - err := bt.DeleteEpoch(i) - if err != nil { - utils.LogFatal(err, "deleting epoch error", 0) - } - - data, err := rpcClient.GetEpochData(uint64(i), true) - if err != nil { - utils.LogFatal(err, "getting epoch data error", 0) - } - - g := new(errgroup.Group) - - g.Go(func() error { - err = db.BigtableClient.SaveProposalAssignments(i, data.ValidatorAssignmentes.ProposerAssignments) - if err != nil { - return fmt.Errorf("error exporting proposal assignments to bigtable: %v", err) - } - return nil - }) - g.Go(func() error { - err = db.BigtableClient.SaveAttestationDuties(data.AttestationDuties) - if err != nil { - return fmt.Errorf("error exporting attestations to bigtable: %v", err) - } - return nil - }) - g.Go(func() error { - err = db.BigtableClient.SaveProposals(data.Blocks) - if err != nil { - return fmt.Errorf("error exporting proposals to bigtable: %v", err) - } - return nil - }) - g.Go(func() error { - err = db.BigtableClient.SaveSyncComitteeDuties(data.SyncDuties) - if err != nil { - return fmt.Errorf("error exporting sync committee duties to bigtable: %v", err) - } - return nil - }) - - err = g.Wait() - - if err != nil { - utils.LogFatal(err, "wait group error", 0) - } + exporter.ExportEpoch(i, rpcClient) } } func monitor(configPath string) { - cfg := &types.Config{} err := utils.ReadConfig(cfg, configPath) if err != nil { @@ -148,47 +100,7 @@ func monitor(configPath string) { for i := head.FinalizedEpoch; i <= head.HeadEpoch; i++ { logrus.Infof("exporting epoch %v", i) - data, err := rpcClient.GetEpochData(i, true) - if err != nil { - utils.LogFatal(err, "getting epoch data error", 0) - } - - g := new(errgroup.Group) - - g.Go(func() error { - err = db.BigtableClient.SaveProposalAssignments(i, data.ValidatorAssignmentes.ProposerAssignments) - if err != nil { - return fmt.Errorf("error exporting proposal assignments to bigtable: %v", err) - } - return nil - }) - g.Go(func() error { - err = db.BigtableClient.SaveAttestationDuties(data.AttestationDuties) - if err != nil { - return fmt.Errorf("error exporting attestations to bigtable: %v", err) - } - return nil - }) - g.Go(func() error { - err = db.BigtableClient.SaveProposals(data.Blocks) - if err != nil { - return fmt.Errorf("error exporting proposals to bigtable: %v", err) - } - return nil - }) - g.Go(func() error { - err = db.BigtableClient.SaveSyncComitteeDuties(data.SyncDuties) - if err != nil { - return fmt.Errorf("error exporting sync committee duties to bigtable: %v", err) - } - return nil - }) - - err = g.Wait() - - if err != nil { - utils.LogFatal(err, "wait group error", 0) - } + exporter.ExportEpoch(i, rpcClient) } current = head.HeadEpoch } diff --git a/exporter/exporter.go b/exporter/exporter.go index 5a6f7dd24f..24572b8e4f 100644 --- a/exporter/exporter.go +++ b/exporter/exporter.go @@ -564,13 +564,6 @@ func ExportEpoch(epoch uint64, client rpc.Client) error { } return nil }) - // g.Go(func() error { - // err = db.BigtableClient.SaveAttestationAssignments(epoch, data.ValidatorAssignmentes.AttestorAssignments) - // if err != nil { - // return fmt.Errorf("error exporting attestation assignments to bigtable: %v", err) - // } - // return nil - // }) g.Go(func() error { err = db.BigtableClient.SaveProposalAssignments(epoch, data.ValidatorAssignmentes.ProposerAssignments) if err != nil { From 1f30fcc6fd40012938502d01164e485510e12f1f Mon Sep 17 00:00:00 2001 From: peter <1674920+peterbitfly@users.noreply.github.com> Date: Mon, 11 Sep 2023 11:53:07 +0200 Subject: [PATCH 095/106] (BIDS-2379) update types --- db/bigtable.go | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/db/bigtable.go b/db/bigtable.go index 2613f28ec6..6aa5cd15bc 100644 --- a/db/bigtable.go +++ b/db/bigtable.go @@ -984,7 +984,7 @@ func (bigtable *Bigtable) GetValidatorAttestationHistory(validators []uint64, st g, gCtx := errgroup.WithContext(ctx) g.SetLimit(concurrency) - attestationsMap := make(map[uint64]map[uint64][]*types.ValidatorAttestation) + attestationsMap := make(map[types.ValidatorIndex]map[types.Slot][]*types.ValidatorAttestation) for i := 0; i < len(validators); i += batchSize { @@ -1026,15 +1026,15 @@ func (bigtable *Bigtable) GetValidatorAttestationHistory(validators []uint64, st } resMux.Lock() - if attestationsMap[validator] == nil { - attestationsMap[validator] = make(map[uint64][]*types.ValidatorAttestation) + if attestationsMap[types.ValidatorIndex(validator)] == nil { + attestationsMap[types.ValidatorIndex(validator)] = make(map[types.Slot][]*types.ValidatorAttestation) } - if attestationsMap[validator][attesterSlot] == nil { - attestationsMap[validator][attesterSlot] = make([]*types.ValidatorAttestation, 0) + if attestationsMap[types.ValidatorIndex(validator)][types.Slot(attesterSlot)] == nil { + attestationsMap[types.ValidatorIndex(validator)][types.Slot(attesterSlot)] = make([]*types.ValidatorAttestation, 0) } - attestationsMap[validator][attesterSlot] = append(attestationsMap[validator][attesterSlot], &types.ValidatorAttestation{ + attestationsMap[types.ValidatorIndex(validator)][types.Slot(attesterSlot)] = append(attestationsMap[types.ValidatorIndex(validator)][types.Slot(attesterSlot)], &types.ValidatorAttestation{ InclusionSlot: inclusionSlot, Status: status, }) @@ -1083,8 +1083,8 @@ func (bigtable *Bigtable) GetValidatorAttestationHistory(validators []uint64, st // Convert the attestationsMap info to the return format // Set the delay of the inclusionSlot for validator, attestations := range attestationsMap { - if res[validator] == nil { - res[validator] = make([]*types.ValidatorAttestation, 0) + if res[uint64(validator)] == nil { + res[uint64(validator)] = make([]*types.ValidatorAttestation, 0) } for attesterSlot, att := range attestations { currentAttInfo := att[0] @@ -1100,18 +1100,18 @@ func (bigtable *Bigtable) GetValidatorAttestationHistory(validators []uint64, st } missedSlotsCount := uint64(0) - for slot := attesterSlot + 1; slot < currentAttInfo.InclusionSlot; slot++ { + for slot := uint64(attesterSlot) + 1; slot < currentAttInfo.InclusionSlot; slot++ { if missedSlotsMap[slot] || orphanedSlotsMap[slot] { missedSlotsCount++ } } - currentAttInfo.Index = validator - currentAttInfo.Epoch = attesterSlot / utils.Config.Chain.Config.SlotsPerEpoch + currentAttInfo.Index = uint64(validator) + currentAttInfo.Epoch = uint64(attesterSlot) / utils.Config.Chain.Config.SlotsPerEpoch currentAttInfo.CommitteeIndex = 0 - currentAttInfo.AttesterSlot = attesterSlot - currentAttInfo.Delay = int64(currentAttInfo.InclusionSlot - attesterSlot - missedSlotsCount - 1) + currentAttInfo.AttesterSlot = uint64(attesterSlot) + currentAttInfo.Delay = int64(currentAttInfo.InclusionSlot - uint64(attesterSlot) - missedSlotsCount - 1) - res[validator] = append(res[validator], currentAttInfo) + res[uint64(validator)] = append(res[uint64(validator)], currentAttInfo) } } From 150bfa89b67c1dab30277f30eff6199932542075 Mon Sep 17 00:00:00 2001 From: peter <1674920+peterbitfly@users.noreply.github.com> Date: Mon, 11 Sep 2023 13:04:04 +0200 Subject: [PATCH 096/106] (BIDS-2379) use correct constant for reversedPaddedEpoch --- db/bigtable.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/db/bigtable.go b/db/bigtable.go index 6aa5cd15bc..f48c529ef9 100644 --- a/db/bigtable.go +++ b/db/bigtable.go @@ -2162,7 +2162,7 @@ func (bigtable *Bigtable) reversePaddedUserID(userID uint64) string { } func (bigtable *Bigtable) reversedPaddedEpoch(epoch uint64) string { - return fmt.Sprintf("%09d", MAX_BLOCK_NUMBER-epoch) + return fmt.Sprintf("%09d", MAX_EPOCH-epoch) } func (bigtable *Bigtable) reversedPaddedSlot(slot uint64) string { From d3072f3984cb9bb9e95f22d0b76b513fc0d2677e Mon Sep 17 00:00:00 2001 From: peter <1674920+peterbitfly@users.noreply.github.com> Date: Mon, 11 Sep 2023 13:15:41 +0200 Subject: [PATCH 097/106] (BIDS-2379) remove dependency on the fdu for the explorer --- exporter/sync_committees_count.go | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/exporter/sync_committees_count.go b/exporter/sync_committees_count.go index 05787f12fc..6d1397eaa1 100644 --- a/exporter/sync_committees_count.go +++ b/exporter/sync_committees_count.go @@ -2,7 +2,6 @@ package exporter import ( "eth2-exporter/db" - "eth2-exporter/services" "eth2-exporter/utils" "fmt" "time" @@ -27,7 +26,12 @@ func exportSyncCommitteesCount() error { return err } - currentPeriod := utils.SyncPeriodOfEpoch(services.LatestFinalizedEpoch()) + latestFinalizedEpoch, err := db.GetLatestFinalizedEpoch() + if err != nil { + logger.Errorf("error retrieving latest exported finalized epoch from the database: %v", err) + } + + currentPeriod := utils.SyncPeriodOfEpoch(latestFinalizedEpoch) firstPeriod := utils.SyncPeriodOfEpoch(utils.Config.Chain.Config.AltairForkEpoch) dbPeriod := uint64(0) From 4cd0f4c77a14d4a5dd39fb75e88345038bbc569b Mon Sep 17 00:00:00 2001 From: peter <1674920+peterbitfly@users.noreply.github.com> Date: Mon, 11 Sep 2023 14:02:37 +0200 Subject: [PATCH 098/106] (BIDS-2379) fix wrong index --- db/bigtable.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/db/bigtable.go b/db/bigtable.go index f48c529ef9..c409ef26c9 100644 --- a/db/bigtable.go +++ b/db/bigtable.go @@ -1713,7 +1713,7 @@ func (bigtable *Bigtable) GetValidatorProposalHistory(validators []uint64, start for _, ri := range r[PROPOSALS_FAMILY] { keySplit := strings.Split(r.Key(), ":") - proposalSlot, err := strconv.ParseUint(keySplit[3], 10, 64) + proposalSlot, err := strconv.ParseUint(keySplit[4], 10, 64) if err != nil { logger.Errorf("error parsing slot from row key %v: %v", r.Key(), err) return false From 4a60c6ee6950f71a5386fddbdadf8373b3c22c44 Mon Sep 17 00:00:00 2001 From: peter <1674920+peterbitfly@users.noreply.github.com> Date: Tue, 12 Sep 2023 09:15:00 +0200 Subject: [PATCH 099/106] (BIDS-2379) address cr issues --- cmd/migrations/bigtable/main.go | 2 -- db/bigtable.go | 8 +++--- db/bigtable_eth1.go | 2 +- ...20230912091400_remove_chain_head_table.sql | 26 +++++++++++++++++++ 4 files changed, 31 insertions(+), 7 deletions(-) create mode 100644 db/migrations/20230912091400_remove_chain_head_table.sql diff --git a/cmd/migrations/bigtable/main.go b/cmd/migrations/bigtable/main.go index 60359f3e58..2bf34869e5 100644 --- a/cmd/migrations/bigtable/main.go +++ b/cmd/migrations/bigtable/main.go @@ -55,8 +55,6 @@ func main() { } for i := *start; i <= *end; i++ { - i := i - logrus.Infof("exporting epoch %v", i) exporter.ExportEpoch(i, rpcClient) } diff --git a/db/bigtable.go b/db/bigtable.go index c409ef26c9..2eeba7da84 100644 --- a/db/bigtable.go +++ b/db/bigtable.go @@ -1054,7 +1054,7 @@ func (bigtable *Bigtable) GetValidatorAttestationHistory(validators []uint64, st // Find all missed and orphaned slots slots := []uint64{} - maxSlot := (endEpoch + 1) * utils.Config.Chain.Config.SlotsPerEpoch + maxSlot := ((endEpoch + 1) * utils.Config.Chain.Config.SlotsPerEpoch) - 1 for slot := startEpoch * utils.Config.Chain.Config.SlotsPerEpoch; slot <= maxSlot; slot++ { slots = append(slots, slot) } @@ -2018,7 +2018,7 @@ func (bigtable *Bigtable) getValidatorsEpochRanges(validatorIndices []uint64, pr startEpoch = 0 } - ranges := make(gcp_bigtable.RowRangeList, 0, int((endEpoch-startEpoch))*len(validatorIndices)+1) + ranges := make(gcp_bigtable.RowRangeList, 0, int((endEpoch-startEpoch))*len(validatorIndices)) for _, validatorIndex := range validatorIndices { validatorKey := bigtable.validatorIndexToKey(validatorIndex) @@ -2038,7 +2038,7 @@ func (bigtable *Bigtable) getValidatorsEpochSlotRanges(validatorIndices []uint64 startEpoch = 0 } - ranges := make(gcp_bigtable.RowRangeList, 0, int((endEpoch-startEpoch))*len(validatorIndices)+1) + ranges := make(gcp_bigtable.RowRangeList, 0, int((endEpoch-startEpoch))*len(validatorIndices)) for _, validatorIndex := range validatorIndices { validatorKey := bigtable.validatorIndexToKey(validatorIndex) @@ -2060,7 +2060,7 @@ func (bigtable *Bigtable) getValidatorSlotRanges(validatorIndices []uint64, pref startEpoch := utils.EpochOfSlot(startSlot) endEpoch := utils.EpochOfSlot(endSlot) - ranges := make(gcp_bigtable.RowRangeList, 0, len(validatorIndices)+1) + ranges := make(gcp_bigtable.RowRangeList, 0, len(validatorIndices)) for _, validatorIndex := range validatorIndices { validatorKey := bigtable.validatorIndexToKey(validatorIndex) diff --git a/db/bigtable_eth1.go b/db/bigtable_eth1.go index e93b3cedef..bd258130f8 100644 --- a/db/bigtable_eth1.go +++ b/db/bigtable_eth1.go @@ -112,7 +112,7 @@ func (bigtable *Bigtable) GetMetadatTable() *gcp_bigtable.Table { } func (bigtable *Bigtable) SaveBlock(block *types.Eth1Block) error { - ctx, cancel := context.WithTimeout(context.Background(), time.Second*10) + ctx, cancel := context.WithTimeout(context.Background(), time.Second*30) defer cancel() encodedBc, err := proto.Marshal(block) diff --git a/db/migrations/20230912091400_remove_chain_head_table.sql b/db/migrations/20230912091400_remove_chain_head_table.sql new file mode 100644 index 0000000000..a9658f7f45 --- /dev/null +++ b/db/migrations/20230912091400_remove_chain_head_table.sql @@ -0,0 +1,26 @@ +-- +goose Up +-- +goose StatementBegin +SELECT 'delete chain_head table'; +DROP TABLE IF EXISTS chain_head; +-- +goose StatementEnd + +-- +goose Down +-- +goose StatementBegin +SELECT 'create chain_head table'; +CREATE TABLE IF NOT EXISTS + chain_head ( + finalized_block_root bytea, + finalized_epoch INT, + finalized_slot INT, + head_block_root bytea, + head_epoch INT, + head_slot INT, + justified_block_root bytea, + justified_epoch INT, + justified_slot INT, + previous_justified_block_root bytea, + previous_justified_epoch INT, + previous_justified_slot INT, + PRIMARY KEY (finalized_epoch) + ); +-- +goose StatementEnd From 17618d8e7a4b9d213aa663a63f745edc810c7b70 Mon Sep 17 00:00:00 2001 From: peter <1674920+peterbitfly@users.noreply.github.com> Date: Tue, 12 Sep 2023 09:41:03 +0200 Subject: [PATCH 100/106] (BIDS-2379) improve map allocations --- db/bigtable.go | 4 ++-- db/bigtable_eth1.go | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/db/bigtable.go b/db/bigtable.go index 2eeba7da84..592972f236 100644 --- a/db/bigtable.go +++ b/db/bigtable.go @@ -2018,7 +2018,7 @@ func (bigtable *Bigtable) getValidatorsEpochRanges(validatorIndices []uint64, pr startEpoch = 0 } - ranges := make(gcp_bigtable.RowRangeList, 0, int((endEpoch-startEpoch))*len(validatorIndices)) + ranges := make(gcp_bigtable.RowRangeList, 0, int((endEpoch-startEpoch+1))*len(validatorIndices)) for _, validatorIndex := range validatorIndices { validatorKey := bigtable.validatorIndexToKey(validatorIndex) @@ -2038,7 +2038,7 @@ func (bigtable *Bigtable) getValidatorsEpochSlotRanges(validatorIndices []uint64 startEpoch = 0 } - ranges := make(gcp_bigtable.RowRangeList, 0, int((endEpoch-startEpoch))*len(validatorIndices)) + ranges := make(gcp_bigtable.RowRangeList, 0, int((endEpoch-startEpoch+1))*len(validatorIndices)) for _, validatorIndex := range validatorIndices { validatorKey := bigtable.validatorIndexToKey(validatorIndex) diff --git a/db/bigtable_eth1.go b/db/bigtable_eth1.go index bd258130f8..39fc68933d 100644 --- a/db/bigtable_eth1.go +++ b/db/bigtable_eth1.go @@ -775,6 +775,7 @@ func (bigtable *Bigtable) IndexEventsWithTransformers(start, end int64, transfor close(stream) }(blocksChan) subG := new(errgroup.Group) + subG.SetLimit(int(concurrency)) for b := range blocksChan { block := b subG.Go(func() error { From e4fc24d656515e863f8d16cf19b766a002504caf Mon Sep 17 00:00:00 2001 From: peter <1674920+peterbitfly@users.noreply.github.com> Date: Tue, 12 Sep 2023 10:01:55 +0200 Subject: [PATCH 101/106] (BIDS-2379) add holesky config --- config/holesky.config.yml | 117 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 117 insertions(+) create mode 100644 config/holesky.config.yml diff --git a/config/holesky.config.yml b/config/holesky.config.yml new file mode 100644 index 0000000000..a6bfd87ade --- /dev/null +++ b/config/holesky.config.yml @@ -0,0 +1,117 @@ +# Extends the mainnet preset +PRESET_BASE: 'mainnet' +CONFIG_NAME: holesky + +# Genesis +# --------------------------------------------------------------- +# `2**14` (= 16,384) +MIN_GENESIS_ACTIVE_VALIDATOR_COUNT: 16384 +# Sep-15-2023 13:55:00 +UTC +MIN_GENESIS_TIME: 1694786100 +GENESIS_FORK_VERSION: 0x00017000 +# Genesis delay 5 mins +GENESIS_DELAY: 300 + + +# Forking +# --------------------------------------------------------------- +# Some forks are disabled for now: +# - These may be re-assigned to another fork-version later +# - Temporarily set to max uint64 value: 2**64 - 1 + +# Altair +ALTAIR_FORK_VERSION: 0x10017000 +ALTAIR_FORK_EPOCH: 0 +# Merge +BELLATRIX_FORK_VERSION: 0x20017000 +BELLATRIX_FORK_EPOCH: 0 +TERMINAL_TOTAL_DIFFICULTY: 0 +TERMINAL_BLOCK_HASH: 0x0000000000000000000000000000000000000000000000000000000000000000 +TERMINAL_BLOCK_HASH_ACTIVATION_EPOCH: 18446744073709551615 + +# Capella +CAPELLA_FORK_VERSION: 0x30017000 +CAPELLA_FORK_EPOCH: 256 + +# DENEB +DENEB_FORK_VERSION: 0x40017000 +DENEB_FORK_EPOCH: 18446744073709551615 + +# Time parameters +# --------------------------------------------------------------- +# 12 seconds +SECONDS_PER_SLOT: 12 +# 14 (estimate from Eth1 mainnet) +SECONDS_PER_ETH1_BLOCK: 14 +# 2**8 (= 256) epochs ~27 hours +MIN_VALIDATOR_WITHDRAWABILITY_DELAY: 256 +# 2**8 (= 256) epochs ~27 hours +SHARD_COMMITTEE_PERIOD: 256 +# 2**11 (= 2,048) Eth1 blocks ~8 hours +ETH1_FOLLOW_DISTANCE: 2048 + + +# Validator cycle +# --------------------------------------------------------------- +# 2**2 (= 4) +INACTIVITY_SCORE_BIAS: 4 +# 2**4 (= 16) +INACTIVITY_SCORE_RECOVERY_RATE: 16 +# 28,000,000,000 Gwei to ensure quicker ejection +EJECTION_BALANCE: 28000000000 +# 2**2 (= 4) +MIN_PER_EPOCH_CHURN_LIMIT: 4 +# 2**16 (= 65,536) +CHURN_LIMIT_QUOTIENT: 65536 + +# Fork choice +# --------------------------------------------------------------- +# 40% +PROPOSER_SCORE_BOOST: 40 + +# Deposit contract +# --------------------------------------------------------------- +DEPOSIT_CHAIN_ID: 17000 +DEPOSIT_NETWORK_ID: 17000 +DEPOSIT_CONTRACT_ADDRESS: 0x4242424242424242424242424242424242424242 + +# Networking +# --------------------------------------------------------------- +# `10 * 2**20` (= 10485760, 10 MiB) +GOSSIP_MAX_SIZE: 10485760 +# `2**10` (= 1024) +MAX_REQUEST_BLOCKS: 1024 +# `2**8` (= 256) +EPOCHS_PER_SUBNET_SUBSCRIPTION: 256 +# `MIN_VALIDATOR_WITHDRAWABILITY_DELAY + CHURN_LIMIT_QUOTIENT // 2` (= 33024, ~5 months) +MIN_EPOCHS_FOR_BLOCK_REQUESTS: 33024 +# `10 * 2**20` (=10485760, 10 MiB) +MAX_CHUNK_SIZE: 10485760 +# 5s +TTFB_TIMEOUT: 5 +# 10s +RESP_TIMEOUT: 10 +ATTESTATION_PROPAGATION_SLOT_RANGE: 32 +# 500ms +MAXIMUM_GOSSIP_CLOCK_DISPARITY: 500 +MESSAGE_DOMAIN_INVALID_SNAPPY: 0x00000000 +MESSAGE_DOMAIN_VALID_SNAPPY: 0x01000000 +# 2 subnets per node +SUBNETS_PER_NODE: 2 +# 2**8 (= 64) +ATTESTATION_SUBNET_COUNT: 64 +ATTESTATION_SUBNET_EXTRA_BITS: 0 +# ceillog2(ATTESTATION_SUBNET_COUNT) + ATTESTATION_SUBNET_EXTRA_BITS +ATTESTATION_SUBNET_PREFIX_BITS: 6 + +# Deneb +# `2**7` (=128) +MAX_REQUEST_BLOCKS_DENEB: 128 +# MAX_REQUEST_BLOCKS_DENEB * MAX_BLOBS_PER_BLOCK +MAX_REQUEST_BLOB_SIDECARS: 768 +# `2**12` (= 4096 epochs, ~18 days) +MIN_EPOCHS_FOR_BLOB_SIDECARS_REQUESTS: 4096 +# `6` +BLOB_SIDECAR_SUBNET_COUNT: 6 +# `uint64(6)` +MAX_BLOBS_PER_BLOCK: 6 From 0055283e58f814fb96a034c36473c9dc287a6b11 Mon Sep 17 00:00:00 2001 From: peter <1674920+peterbitfly@users.noreply.github.com> Date: Tue, 12 Sep 2023 10:01:56 +0200 Subject: [PATCH 102/106] (BIDS-2379) add holesky chain config --- config/config.go | 3 +++ config/{holesky.config.yml => holesky.chain.yml} | 0 services/charts_updater.go | 2 +- services/services.go | 2 +- utils/utils.go | 2 ++ 5 files changed, 7 insertions(+), 2 deletions(-) rename config/{holesky.config.yml => holesky.chain.yml} (100%) diff --git a/config/config.go b/config/config.go index 0e8d4caf82..5512664bc3 100644 --- a/config/config.go +++ b/config/config.go @@ -22,3 +22,6 @@ var TestnetChainYml string //go:embed gnosis.chain.yml var GnosisChainYml string + +//go:embed holesky.chain.yml +var HoleskyChainYml string diff --git a/config/holesky.config.yml b/config/holesky.chain.yml similarity index 100% rename from config/holesky.config.yml rename to config/holesky.chain.yml diff --git a/services/charts_updater.go b/services/charts_updater.go index a96d502f9c..b620bb6bed 100644 --- a/services/charts_updater.go +++ b/services/charts_updater.go @@ -74,7 +74,7 @@ func LatestChartsPageData() []*types.ChartsPageDataChart { } func chartsPageDataUpdater(wg *sync.WaitGroup) { - sleepDuration := time.Second * time.Duration(utils.Config.Chain.Config.SecondsPerSlot) + sleepDuration := time.Hour // only update charts once per hour var prevEpoch uint64 firstun := true diff --git a/services/services.go b/services/services.go index 3f8e7c23be..e5460fb241 100644 --- a/services/services.go +++ b/services/services.go @@ -1499,7 +1499,7 @@ func burnUpdater(wg *sync.WaitGroup) { wg.Done() firstRun = false } - time.Sleep(time.Minute) + time.Sleep(time.Minute * 15) // only update once every 15 minutes } } diff --git a/utils/utils.go b/utils/utils.go index 341339bdda..26b7decac0 100644 --- a/utils/utils.go +++ b/utils/utils.go @@ -417,6 +417,8 @@ func ReadConfig(cfg *types.Config, path string) error { err = yaml.Unmarshal([]byte(config.SepoliaChainYml), &cfg.Chain.Config) case "gnosis": err = yaml.Unmarshal([]byte(config.GnosisChainYml), &cfg.Chain.Config) + case "holesky": + err = yaml.Unmarshal([]byte(config.HoleskyChainYml), &cfg.Chain.Config) default: return fmt.Errorf("tried to set known chain-config, but unknown chain-name") } From e536a0d7c631b55267c8c511da1bdcb247a0b641 Mon Sep 17 00:00:00 2001 From: peter <1674920+peterbitfly@users.noreply.github.com> Date: Tue, 12 Sep 2023 10:56:30 +0200 Subject: [PATCH 103/106] (BIDS-2379) update dependencies --- go.mod | 45 ++-- go.sum | 640 +++++++++++---------------------------------------------- 2 files changed, 136 insertions(+), 549 deletions(-) diff --git a/go.mod b/go.mod index 77c2a26e08..f0edc9593e 100644 --- a/go.mod +++ b/go.mod @@ -12,6 +12,7 @@ require ( github.com/asaskevich/govalidator v0.0.0-20210307081110-f21760c49a8d github.com/awa/go-iap v1.3.7 github.com/aybabtme/uniplot v0.0.0-20151203143629-039c559e5e7e + github.com/carlmjohnson/requests v0.23.4 github.com/davecgh/go-spew v1.1.1 github.com/ethereum/go-ethereum v1.12.2 github.com/evanw/esbuild v0.8.23 @@ -79,14 +80,11 @@ require ( cloud.google.com/go/compute/metadata v0.2.1 // indirect cloud.google.com/go/iam v0.7.0 // indirect cloud.google.com/go/longrunning v0.3.0 // indirect - github.com/FactomProject/basen v0.0.0-20150613233007-fe3947df716e // indirect - github.com/FactomProject/btcutilecc v0.0.0-20130527213604-d3a63a5752ec // indirect - github.com/FactomProject/go-bip32 v0.3.5 // indirect - github.com/FactomProject/go-bip39 v0.3.5 // indirect - github.com/FactomProject/go-bip44 v0.0.0-20190306062959-b541a96d8da9 // indirect + github.com/DataDog/zstd v1.5.2 // indirect + github.com/VictoriaMetrics/fastcache v1.6.0 // indirect github.com/alessio/shellescape v1.4.1 // indirect - github.com/antlr/antlr4/runtime/Go/antlr v0.0.0-20210716071054-a231a1a7f1cc // indirect - github.com/carlmjohnson/requests v0.23.4 // indirect + github.com/allegro/bigcache v1.2.1 // indirect + github.com/bits-and-blooms/bitset v1.7.0 // indirect github.com/census-instrumentation/opencensus-proto v0.2.1 // indirect github.com/cncf/udpa/go v0.0.0-20210930031921-04548b0d99d4 // indirect github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1 // indirect @@ -109,15 +107,14 @@ require ( github.com/gofrs/flock v0.8.1 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/googleapis/enterprise-certificate-proxy v0.2.0 // indirect - github.com/hashicorp/hcl v1.0.0 // indirect - github.com/holiman/uint256 v1.2.0 // indirect + github.com/holiman/bloomfilter/v2 v2.0.3 // indirect + github.com/holiman/uint256 v1.2.3 // indirect github.com/imdario/mergo v0.3.13 // indirect - github.com/inconshreveable/mousetrap v1.0.0 // indirect - github.com/ipfs/go-cid v0.3.2 // indirect - github.com/karalabe/usb v0.0.2 // indirect - github.com/kilic/bls12-381 v0.1.0 // indirect - github.com/magiconair/properties v1.8.5 // indirect - github.com/miekg/dns v1.1.50 // indirect + github.com/ipfs/go-cid v0.4.1 // indirect + github.com/kr/pretty v0.3.1 // indirect + github.com/kr/text v0.2.0 // indirect + github.com/mattn/go-runewidth v0.0.14 // indirect + github.com/mattn/go-sqlite3 v1.11.0 // indirect github.com/mitchellh/go-homedir v1.1.0 // indirect github.com/mmcloughlin/addchain v0.4.0 // indirect github.com/multiformats/go-base32 v0.1.0 // indirect @@ -127,13 +124,7 @@ require ( github.com/multiformats/go-varint v0.0.7 // indirect github.com/olekukonko/tablewriter v0.0.5 // indirect github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 // indirect - github.com/pborman/uuid v1.2.1 // indirect - github.com/pelletier/go-toml v1.9.3 // indirect - github.com/protolambda/bls12-381-util v0.0.0-20210720105258-a772f2aac13e // indirect - github.com/protolambda/eth2-val-tools v0.1.2-0.20230223182759-4bf01453537a // indirect - github.com/protolambda/go-keystorev4 v0.0.0-20211007151826-f20444f6d564 // indirect github.com/protolambda/zssz v0.1.5 // indirect - github.com/protolambda/ztyp v0.2.2 // indirect github.com/prysmaticlabs/fastssz v0.0.0-20221107182844-78142813af44 // indirect github.com/prysmaticlabs/gohashtree v0.0.2-alpha // indirect github.com/rivo/uniseg v0.4.3 // indirect @@ -141,23 +132,11 @@ require ( github.com/rs/cors v1.8.0 // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect github.com/spaolacci/murmur3 v1.1.0 // indirect - github.com/spf13/afero v1.6.0 // indirect - github.com/spf13/cast v1.4.0 // indirect - github.com/spf13/cobra v1.2.1 // indirect - github.com/spf13/jwalterweatherman v1.1.0 // indirect - github.com/spf13/pflag v1.0.5 // indirect - github.com/spf13/viper v1.8.1 // indirect - github.com/subosito/gotenv v1.2.0 // indirect - github.com/tyler-smith/go-bip39 v1.1.0 // indirect github.com/urfave/cli v1.22.12 // indirect - github.com/wealdtech/ethereal v0.0.0-20211110093529-ed2beb2b4a38 // indirect github.com/wealdtech/go-bytesutil v1.2.1 // indirect - github.com/wealdtech/go-erc1820 v1.2.3 // indirect github.com/wealdtech/go-merkletree v1.0.1-0.20190605192610-2bb163c2ea2a // indirect github.com/wealdtech/go-multicodec v1.4.0 // indirect - github.com/wealdtech/go-string2eth v1.1.0 // indirect github.com/yusufpapurcu/wmi v1.2.2 // indirect - golang.org/x/mod v0.9.0 // indirect google.golang.org/genproto v0.0.0-20221118155620-16455021b5e6 // indirect google.golang.org/grpc v1.52.3 // indirect lukechampine.com/blake3 v1.2.1 // indirect diff --git a/go.sum b/go.sum index d9e4871390..d73ec11112 100644 --- a/go.sum +++ b/go.sum @@ -1,13 +1,11 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= -cloud.google.com/go v0.43.0/go.mod h1:BOSR3VbTLkk6FDC/TcffxP4NF/FFBGA5ku+jvKOP7pg= cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU= cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc= cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0= cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6To= -cloud.google.com/go v0.51.0/go.mod h1:hWtGJ6gnXH+KgDv+V0zFGDvpi07n3z8ZNj3T1RW0Gcw= cloud.google.com/go v0.52.0/go.mod h1:pXajvRH/6o3+F9jDHZWQ5PbGhn+o8w9qiu/CffaVdO4= cloud.google.com/go v0.53.0/go.mod h1:fp/UouUEsRkN6ryDKNW/Upv/JBKnv6WDthjR6+vze6M= cloud.google.com/go v0.54.0/go.mod h1:1rq2OEkV3YMf6n/9ZvGWI3GWw0VoqH/1x2nd8Is/bPc= @@ -17,9 +15,6 @@ cloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOY cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHObY= cloud.google.com/go v0.72.0/go.mod h1:M+5Vjvlc2wnp6tjzE102Dw08nGShTscUx2nZMufOKPI= cloud.google.com/go v0.74.0/go.mod h1:VV1xSbzvo+9QJOxLDaJfTjx5e+MePCpCWwvftOeQmWk= -cloud.google.com/go v0.78.0/go.mod h1:QjdrLG0uq+YwhjoVOLsS1t7TW8fs36kLs4XO5R5ECHg= -cloud.google.com/go v0.79.0/go.mod h1:3bzgcEeQlzbuEAYu4mrWhKqWjmpprinYgKJLgKHnbb8= -cloud.google.com/go v0.81.0/go.mod h1:mk/AM35KwGk/Nm2YSeZbxXdrNK3KZOYHmLkOqC2V6E0= cloud.google.com/go v0.105.0 h1:DNtEKRBAAzeS4KyIory52wWHuClNaXJ5x1F7xa4q+5Y= cloud.google.com/go v0.105.0/go.mod h1:PrLgOJNe5nfE9UMxKxgXj4mD3voiP+YQ6gdt6KMFOKM= cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= @@ -28,7 +23,6 @@ cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvf cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg= cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc= cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ= -cloud.google.com/go/bigtable v1.2.0/go.mod h1:JcVAOl45lrTmQfLj7T6TxyMzIN/3FGGcFm+2xVAli2o= cloud.google.com/go/bigtable v1.16.0 h1:sqJhhslzQOag49Mf2/uH3+u+NdfpPX0gjKAcgYpRUCU= cloud.google.com/go/bigtable v1.16.0/go.mod h1:6f7WVXfeZaJz0xevUZoTA1s8sTmmrQqIAkRDVEHVg7I= cloud.google.com/go/compute v1.12.1 h1:gKVJMEyqV5c/UnpzjjQbo3Rjvvqpr9B1DFSbJC4OXr0= @@ -57,40 +51,19 @@ cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RX cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= cloud.google.com/go/storage v1.27.0 h1:YOO045NZI9RKfCj1c5A/ZtuuENUc8OAW+gHdGnDgyMQ= cloud.google.com/go/storage v1.27.0/go.mod h1:x9DOL8TK/ygDUMieqwfhdpQryTeEkhGKMi80i/iqR2s= -collectd.org v0.3.0/go.mod h1:A/8DzQBkF6abtvrT2j/AU/4tiBgJWYyh0y/oB/4MlWE= contrib.go.opencensus.io/exporter/jaeger v0.2.1 h1:yGBYzYMewVL0yO9qqJv3Z5+IRhPdU7e9o/2oKpX4YvI= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= firebase.google.com/go v3.13.0+incompatible h1:3TdYC3DDi6aHn20qoRkxwGqNgdjtblwVAyRLQwGn/+4= firebase.google.com/go v3.13.0+incompatible/go.mod h1:xlah6XbEyW6tbfSklcfe5FHJIwjt8toICdV5Wh9ptHs= -github.com/Azure/azure-pipeline-go v0.2.1/go.mod h1:UGSo8XybXnIGZ3epmeBw7Jdz+HiUVpqIlpz/HKHylF4= -github.com/Azure/azure-pipeline-go v0.2.2/go.mod h1:4rQ/NZncSvGqNkkOsNpOU1tgoNuIlp9AfUH5G1tvCHc= -github.com/Azure/azure-storage-blob-go v0.7.0/go.mod h1:f9YQKtsG1nMisotuTPpO0tjNuEjKRYAcJU8/ydDI++4= -github.com/Azure/go-autorest/autorest v0.9.0/go.mod h1:xyHB1BMZT0cuDHU7I0+g046+BFDTQ8rEZB0s4Yfa6bI= -github.com/Azure/go-autorest/autorest/adal v0.5.0/go.mod h1:8Z9fGy2MpX0PvDjB1pEgQTmVqjGhiHBW7RJJEciWzS0= -github.com/Azure/go-autorest/autorest/adal v0.8.0/go.mod h1:Z6vX6WXXuyieHAXwMj0S6HY6e6wcHn37qQMBQlvY3lc= -github.com/Azure/go-autorest/autorest/date v0.1.0/go.mod h1:plvfp3oPSKwf2DNjlBjWF/7vwR+cUD/ELuzDCXwHUVA= -github.com/Azure/go-autorest/autorest/date v0.2.0/go.mod h1:vcORJHLJEh643/Ioh9+vPmf1Ij9AEBM5FuBIXLmIy0g= -github.com/Azure/go-autorest/autorest/mocks v0.1.0/go.mod h1:OTyCOPRA2IgIlWxVYxBee2F5Gr4kF2zd2J5cFRaIDN0= -github.com/Azure/go-autorest/autorest/mocks v0.2.0/go.mod h1:OTyCOPRA2IgIlWxVYxBee2F5Gr4kF2zd2J5cFRaIDN0= -github.com/Azure/go-autorest/autorest/mocks v0.3.0/go.mod h1:a8FDP3DYzQ4RYfVAxAN3SVSiiO77gL2j2ronKKP0syM= -github.com/Azure/go-autorest/logger v0.1.0/go.mod h1:oExouG+K6PryycPJfVSxi/koC6LSNgds39diKLz7Vrc= -github.com/Azure/go-autorest/tracing v0.5.0/go.mod h1:r/s2XiOKccPW3HrqB+W0TQzfbtp2fGCgRFtBroKn4Dk= +github.com/AndreasBriese/bbloom v0.0.0-20190306092124-e2d15f34fcf9/go.mod h1:bOvUY6CB00SOBii9/FifXqc0awNKxLFCL/+pkDPuyl8= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/toml v1.2.1 h1:9F2/+DoOYIOksmaJFPw1tGFy1eDnIJXg+UHjuD8lTak= github.com/BurntSushi/toml v1.2.1/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= -github.com/DATA-DOG/go-sqlmock v1.3.3/go.mod h1:f/Ixk793poVmq4qj/V1dPUg2JEAKC73Q5eFN3EC/SaM= +github.com/CloudyKit/fastprinter v0.0.0-20200109182630-33d98a066a53/go.mod h1:+3IMCy2vIlbG1XG/0ggNQv0SvxCAIpPM5b1nCz56Xno= +github.com/CloudyKit/jet/v3 v3.0.0/go.mod h1:HKQPgSJmdK8hdoAbKUUWajkHyHo4RaU5rMdUywE7VMo= github.com/DataDog/zstd v1.5.2 h1:vUG4lAyuPCXO0TLbXvPv7EB7cNK1QV/luu55UHLrrn8= -github.com/FactomProject/basen v0.0.0-20150613233007-fe3947df716e h1:ahyvB3q25YnZWly5Gq1ekg6jcmWaGj/vG/MhF4aisoc= -github.com/FactomProject/basen v0.0.0-20150613233007-fe3947df716e/go.mod h1:kGUqhHd//musdITWjFvNTHn90WG9bMLBEPQZ17Cmlpw= -github.com/FactomProject/btcutilecc v0.0.0-20130527213604-d3a63a5752ec h1:1Qb69mGp/UtRPn422BH4/Y4Q3SLUrD9KHuDkm8iodFc= -github.com/FactomProject/btcutilecc v0.0.0-20130527213604-d3a63a5752ec/go.mod h1:CD8UlnlLDiqb36L110uqiP2iSflVjx9g/3U9hCI4q2U= -github.com/FactomProject/go-bip32 v0.3.5 h1:etsJ4Y/wA7spZl/WG830ogTHQivGcV/8sCrrmkQICNQ= -github.com/FactomProject/go-bip32 v0.3.5/go.mod h1:efm/M7J/CGmQ5dPtGM0GWod5LuyShuFET6oY13168w4= -github.com/FactomProject/go-bip39 v0.3.5 h1:l9g92TeqCkC5NZhm72igTpf5yaYDp3Sy4CvnPYknp6U= -github.com/FactomProject/go-bip39 v0.3.5/go.mod h1:ygPVOtW424QxnJMze9XYDeh4wT19V3iVDOqVUl/USkE= -github.com/FactomProject/go-bip44 v0.0.0-20190306062959-b541a96d8da9 h1:Wprj9FTxqhjgl7e8ZHScGwF5Wc2WrHTbEDoQKciBdhw= -github.com/FactomProject/go-bip44 v0.0.0-20190306062959-b541a96d8da9/go.mod h1:4H/Y1yLgSJvRV4iqilsVhYlqXkPgf1QonpKyt6sdP+Q= +github.com/DataDog/zstd v1.5.2/go.mod h1:g4AWEaM3yOg3HYfnJ3YIawPnVdXJh9QME85blwSAmyw= github.com/Gurpartap/storekit-go v0.0.0-20201205024111-36b6cd5c6a21 h1:HcdvlzaQ4CJfH7xbfJZ3ZHN//BTEpId46iKEMuP3wHE= github.com/Gurpartap/storekit-go v0.0.0-20201205024111-36b6cd5c6a21/go.mod h1:7PODFS++oNZ6khojmPBvkrDeFO/hrc3jmvWvQAOXorw= github.com/Joker/hpp v1.0.0/go.mod h1:8x5n+M1Hp5hC0g8okX3sR3vFQwynaX/UgSOM9MeBKzY= @@ -99,17 +72,10 @@ github.com/KyleBanks/depth v1.2.1/go.mod h1:jzSb9d0L43HxTQfT+oSA1EEp2q+ne2uh6Xge github.com/Masterminds/semver/v3 v3.1.1 h1:hLg3sBzpNErnxhQtUy/mmLR2I9foDujNK030IGemrRc= github.com/Masterminds/semver/v3 v3.1.1/go.mod h1:VPu/7SZ7ePZ3QOrcuXROw5FAcLl4a0cBrbBpGY/8hQs= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= -github.com/Shopify/sarama v1.26.1/go.mod h1:NbSGBSSndYaIhRcBtY9V0U7AyH+x71bG668AuWys/yU= -github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMxUHB2q5Ap20/P/eIdh4G0pI= -github.com/StackExchange/wmi v0.0.0-20180116203802-5d049714c4a6/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg= -github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg= -github.com/StackExchange/wmi v0.0.0-20210224194228-fe8f1750fd46/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg= -github.com/StackExchange/wmi v1.2.1/go.mod h1:rcmrprowKIVzvc+NUiLncP2uuArMWLCbu9SBzvHz7e8= -github.com/VictoriaMetrics/fastcache v1.5.7/go.mod h1:ptDBkNMQI4RtmVo8VS/XwRY6RoTu1dAWCbrk+6WsEM8= +github.com/Shopify/goreferrer v0.0.0-20181106222321-ec9c9a553398/go.mod h1:a1uqRtAwp2Xwc6WNPJEufxJ7fx3npB4UV/JOLmbu5I0= github.com/VictoriaMetrics/fastcache v1.6.0 h1:C/3Oi3EiBCqufydp1neRZkqcwmEiuRT9c3fqvvgKm5o= github.com/VictoriaMetrics/fastcache v1.6.0/go.mod h1:0qHz5QP0GMX4pfmMA/zt5RgfNuXJrTP0zS7DqpHGGTw= -github.com/aead/siphash v1.0.1/go.mod h1:Nywa3cDsYNNK3gaciGTWPwHt0wlpNV15vwmswBAUSII= -github.com/ajstarks/svgo v0.0.0-20180226025133-644b8db467af/go.mod h1:K08gAheRH3/J6wwsYMMT4xOr94bZjxIelGM0+d/wbFw= +github.com/ajg/form v1.5.1/go.mod h1:uL1WgH+h2mgNtvBq0339dVnzXdBETtL2LeUXaIv25UY= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= @@ -122,17 +88,9 @@ github.com/alexedwards/scs/redisstore v0.0.0-20230217120314-6b1bedc0f08c/go.mod github.com/alexedwards/scs/v2 v2.5.0 h1:zgxOfNFmiJyXG7UPIuw1g2b9LWBeRLh3PjfB9BDmfL4= github.com/alexedwards/scs/v2 v2.5.0/go.mod h1:ToaROZxyKukJKT/xLcVQAChi5k6+Pn1Gvmdl7h3RRj8= github.com/allegro/bigcache v1.2.1-0.20190218064605-e24eb225f156/go.mod h1:Cb/ax3seSYIx7SuZdm2G2xzfwmv3TPSk2ucNfQESPXM= +github.com/allegro/bigcache v1.2.1 h1:hg1sY1raCwic3Vnsvje6TT7/pnZba83LeFck5NrFKSc= github.com/allegro/bigcache v1.2.1/go.mod h1:Cb/ax3seSYIx7SuZdm2G2xzfwmv3TPSk2ucNfQESPXM= -github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883/go.mod h1:rCTlJbsFo29Kk6CurOXKm700vrz8f0KW0JNfpkRJY/8= github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= -github.com/antlr/antlr4/runtime/Go/antlr v0.0.0-20210716071054-a231a1a7f1cc h1:0LxTNJTbCStbOMoNcQEc6naaWfU7rI0b+Zdns1EI7TA= -github.com/antlr/antlr4/runtime/Go/antlr v0.0.0-20210716071054-a231a1a7f1cc/go.mod h1:F7bn7fEU90QkQ3tnmaTx3LTKLEDqnwWODIYppRQ5hnY= -github.com/apache/arrow/go/arrow v0.0.0-20191024131854-af6fa24be0db/go.mod h1:VTxUBvSJ3s3eHAg65PNgrsn5BtqCRPdmyXh6rAfdxN0= -github.com/aristanetworks/fsnotify v1.4.2/go.mod h1:D/rtu7LpjYM8tRJphJ0hUBYpjai8SfX+aSNsWDTq/Ks= -github.com/aristanetworks/glog v0.0.0-20191112221043-67e8567f59f3/go.mod h1:KASm+qXFKs/xjSoWn30NrWBBvdTTQq+UjkhjEJHfSFA= -github.com/aristanetworks/goarista v0.0.0-20170210015632-ea17b1a17847/go.mod h1:D/tb0zPVXnP7fmsLZjtdUhSsumbK/ij54UXjjVgMGxQ= -github.com/aristanetworks/goarista v0.0.0-20201012165903-2cb20defcd66/go.mod h1:QZe5Yh80Hp1b6JxQdpfSEEe8X7hTyTEZSosSrFf/oJE= -github.com/aristanetworks/splunk-hec-go v0.3.3/go.mod h1:1VHO9r17b0K7WmOlLb9nTk/2YanvOEnLMUgsFrxBROc= github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= @@ -143,16 +101,6 @@ github.com/attestantio/go-eth2-client v0.15.7 h1:0v7+Z2RZ8bNtU/0mfppXzLiYv+6a8pe github.com/attestantio/go-eth2-client v0.15.7/go.mod h1:/Oh6YTuHmHhgLN/ZnQRKHGc7HdIzGlDkI2vjNZvOsvA= github.com/awa/go-iap v1.3.7 h1:ErmeZRa8I4tx+ToAHikpARoAZVSszHWpwyl4FCj/6XA= github.com/awa/go-iap v1.3.7/go.mod h1:Jq6HjuGiT1FXSp92RDmpnW8c9SzmEqp10fE3FrljmBI= -github.com/aws/aws-sdk-go v1.25.48/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= -github.com/aws/aws-sdk-go-v2 v1.2.0/go.mod h1:zEQs02YRBw1DjK0PoJv3ygDYOFTre1ejlJWl8FwAuQo= -github.com/aws/aws-sdk-go-v2/config v1.1.1/go.mod h1:0XsVy9lBI/BCXm+2Tuvt39YmdHwS5unDQmxZOYe8F5Y= -github.com/aws/aws-sdk-go-v2/credentials v1.1.1/go.mod h1:mM2iIjwl7LULWtS6JCACyInboHirisUUdkBPoTHMOUo= -github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.0.2/go.mod h1:3hGg3PpiEjHnrkrlasTfxFqUsZ2GCk/fMUn4CbKgSkM= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.0.2/go.mod h1:45MfaXZ0cNbeuT0KQ1XJylq8A6+OpVV2E5kvY/Kq+u8= -github.com/aws/aws-sdk-go-v2/service/route53 v1.1.1/go.mod h1:rLiOUrPLW/Er5kRcQ7NkwbjlijluLsrIbu/iyl35RO4= -github.com/aws/aws-sdk-go-v2/service/sso v1.1.1/go.mod h1:SuZJxklHxLAXgLTc1iFXbEWkXs7QRTQpCLGaKIprQW0= -github.com/aws/aws-sdk-go-v2/service/sts v1.1.1/go.mod h1:Wi0EBZwiz/K44YliU0EKxqTCJGUfYTWXrrBwkq736bM= -github.com/aws/smithy-go v1.1.0/go.mod h1:EzMw8dbp/YJL4A5/sbhGddag+NPT7q084agLbB9LgIw= github.com/aybabtme/uniplot v0.0.0-20151203143629-039c559e5e7e h1:dSeuFcs4WAJJnswS8vXy7YY1+fdlbVPuEVmDAfqvFOQ= github.com/aybabtme/uniplot v0.0.0-20151203143629-039c559e5e7e/go.mod h1:uh71c5Vc3VNIplXOFXsnDy21T1BepgT32c5X/YPrOyc= github.com/aymerick/raymond v2.0.3-0.20180322193309-b565731e1464+incompatible/go.mod h1:osfaiScAUVup+UC9Nfq76eWqDhXlp+4UYaA8uhTBO6g= @@ -166,39 +114,18 @@ github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kB github.com/bits-and-blooms/bitset v1.7.0 h1:YjAGVd3XmtK9ktAbX8Zg2g2PwLIMjGREZJHlV4j7NEo= github.com/bits-and-blooms/bitset v1.7.0/go.mod h1:gIdJ4wp64HaoK2YrL1Q5/N7Y16edYb8uY+O0FJTyyDA= github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c/go.mod h1:MKsuJmJgSg28kpZDP6UIiPt0e0Oz0kqKNGyRaWEPv84= -github.com/bketelsen/crypt v0.0.4/go.mod h1:aI6NrJ0pMGgvZKL1iVgXLnfIFJtfV+bKCoqOes/6LfM= github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869 h1:DDGfHa7BWjL4YnC6+E63dPcxHo2sUxDIu8g3QgEJdRY= github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869/go.mod h1:Ekp36dRnpXw/yCqJaO+ZrUyxD+3VXMFFr56k5XYrpB4= -github.com/bmizerany/pat v0.0.0-20170815010413-6226ea591a40/go.mod h1:8rLXio+WjiTceGBHIoTvn60HIbs7Hm7bcHjyrSqYB9c= -github.com/boltdb/bolt v1.3.1/go.mod h1:clJnj/oiGkjum5o1McbSZDSLxVThjynRyGBgiAx27Ps= -github.com/btcsuite/btcd v0.0.0-20171128150713-2e60448ffcc6/go.mod h1:Dmm/EzmjnCiweXmzRIAiUWCInVmPgjkzgv5k4tVyXiQ= -github.com/btcsuite/btcd v0.20.1-beta/go.mod h1:wVuoA8VJLEcwgqHBwHmzLRazpKxTv13Px/pDuV7OomQ= -github.com/btcsuite/btcd v0.21.0-beta/go.mod h1:ZSWyehm27aAuS9bvkATT+Xte3hjHZ+MRgMY/8NJ7K94= -github.com/btcsuite/btcd v0.22.0-beta/go.mod h1:9n5ntfhhHQBIhUvlhDvD3Qg6fRUj4jkN0VB8L8svzOA= github.com/btcsuite/btcd v0.23.4 h1:IzV6qqkfwbItOS/sg/aDfPDsjPP8twrCOE2R93hxMlQ= github.com/btcsuite/btcd/btcec/v2 v2.3.2 h1:5n0X6hX0Zk+6omWcihdYvdAlGf2DfasC0GMf7DClJ3U= github.com/btcsuite/btcd/btcec/v2 v2.3.2/go.mod h1:zYzJ8etWJQIv1Ogk7OzpWjowwOdXY1W/17j2MW85J04= github.com/btcsuite/btcd/chaincfg/chainhash v1.0.1 h1:q0rUy8C/TYNBQS1+CGKw68tLOFYSNEs0TFnxxnS9+4U= -github.com/btcsuite/btclog v0.0.0-20170628155309-84c8d2346e9f/go.mod h1:TdznJufoqS23FtqVCzL0ZqgP5MqXbb4fg/WgDys70nA= -github.com/btcsuite/btcutil v0.0.0-20190425235716-9e5f4b9a998d/go.mod h1:+5NJ2+qvTyV9exUAL/rxXi3DcLg2Ts+ymUAY5y4NvMg= -github.com/btcsuite/btcutil v1.0.2/go.mod h1:j9HUFwoQRsZL3V4n+qG+CUnEGHOarIxfC3Le2Yhbcts= -github.com/btcsuite/btcutil v1.0.3-0.20201208143702-a53e38424cce/go.mod h1:0DVlHczLPewLcPGEIeUEzfOJhqGPQ0mJJRDBtD307+o= -github.com/btcsuite/go-socks v0.0.0-20170105172521-4720035b7bfd/go.mod h1:HHNXQzUsZCxOoE+CPiyCTO6x34Zs86zZUiwtpXoGdtg= -github.com/btcsuite/goleveldb v0.0.0-20160330041536-7834afc9e8cd/go.mod h1:F+uVaaLLH7j4eDXPRvw78tMflu7Ie2bzYOH4Y8rRKBY= -github.com/btcsuite/goleveldb v1.0.0/go.mod h1:QiK9vBlgftBg6rWQIj6wFzbPfRjiykIEhBH4obrXJ/I= -github.com/btcsuite/snappy-go v0.0.0-20151229074030-0bdef8d06723/go.mod h1:8woku9dyThutzjeg+3xrA5iCpBRH8XEEg3lh6TiUghc= -github.com/btcsuite/snappy-go v1.0.0/go.mod h1:8woku9dyThutzjeg+3xrA5iCpBRH8XEEg3lh6TiUghc= -github.com/btcsuite/websocket v0.0.0-20150119174127-31079b680792/go.mod h1:ghJtEyQwv5/p4Mg4C0fgbePVuGr935/5ddU9Z3TmDRY= -github.com/btcsuite/winsvc v1.0.0/go.mod h1:jsenWakMcC0zFBFurPLEAyrnc/teJEM1O46fmI40EZs= github.com/bufbuild/buf v0.37.0/go.mod h1:lQ1m2HkIaGOFba6w/aC3KYBHhKEOESP3gaAEpS3dAFM= -github.com/c-bata/go-prompt v0.2.2/go.mod h1:VzqtzE2ksDBcdln8G7mk2RX9QyGjH+OVqOCSiVIqS34= github.com/carlmjohnson/requests v0.23.4 h1:AxcvapfB9RPXLSyvAHk9YJoodQ43ZjzNHj6Ft3tQGdg= github.com/carlmjohnson/requests v0.23.4/go.mod h1:Qzp6tW4DQyainPP+tGwiJTzwxvElTIKm0B191TgTtOA= github.com/census-instrumentation/opencensus-proto v0.2.1 h1:glEXhBS5PSLLv4IXzLA5yPRVX4bilULVyxxbrfOtDAk= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= -github.com/cespare/cp v0.1.0/go.mod h1:SOGHArjBr4JWaSDEVpWpo/hNg6RoKrls6Oh40hiwW+s= github.com/cespare/cp v1.1.1 h1:nCb6ZLdB7NRaqsm91JtQTAme2SKJzXVsdPIPkyJr1MU= -github.com/cespare/cp v1.1.1/go.mod h1:SOGHArjBr4JWaSDEVpWpo/hNg6RoKrls6Oh40hiwW+s= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= @@ -208,9 +135,6 @@ github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWR github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= -github.com/cloudflare/cloudflare-go v0.10.2-0.20190916151808-a80f83b9add9/go.mod h1:1MxXX1Ux4x6mqPmjkUgTP1CdXIBXKX7T+Jk9Gxrmx+U= -github.com/cloudflare/cloudflare-go v0.14.0/go.mod h1:EnwdgGMaFOruiPZRFSgn+TsQ3hQ7C/YWzIGLeu5c304= -github.com/cmars/basen v0.0.0-20150613233007-fe3947df716e/go.mod h1:P13beTBKr5Q18lJe1rIoLUqjM+CB1zYrRg44ZqGuQSA= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= @@ -232,8 +156,12 @@ github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b/go.mod h1:Vz9D github.com/cockroachdb/pebble v0.0.0-20230209160836-829675f94811 h1:ytcWPaNPhNoGMWEhDvS3zToKcDpRsLuRolQJBVGdozk= github.com/cockroachdb/pebble v0.0.0-20230209160836-829675f94811/go.mod h1:Nb5lgvnQ2+oGlE/EyZy4+2/CxRh9KfvCXnag1vtpxVM= github.com/cockroachdb/redact v1.1.3 h1:AKZds10rFSIj7qADf0g46UixK8NNLwWTNdCIGS5wfSQ= -github.com/consensys/bavard v0.1.8-0.20210406032232-f3452dc9b572/go.mod h1:Bpd0/3mZuaj6Sj+PqrmIquiOKy397AKGThQPaGzNXAQ= -github.com/consensys/gnark-crypto v0.4.1-0.20210426202927-39ac3d4b3f1f/go.mod h1:815PAHg3wvysy0SyIqanF8gZ0Y1wjk/hrDHD/iT88+Q= +github.com/cockroachdb/redact v1.1.3/go.mod h1:BVNblN9mBWFyMyqK1k3AAiSxhvhfK2oOZZ2lK+dpvRg= +github.com/codegangsta/inject v0.0.0-20150114235600-33e0aa1cb7c0/go.mod h1:4Zcjuz89kmFXt9morQgcfYZAYZ5n8WHjt81YYWIwtTM= +github.com/consensys/bavard v0.1.13 h1:oLhMLOFGTLdlda/kma4VOJazblc7IM5y5QPd2A/YjhQ= +github.com/consensys/bavard v0.1.13/go.mod h1:9ItSMtA/dXMAiL7BG6bqW2m3NdSEObYWoH223nGHukI= +github.com/consensys/gnark-crypto v0.10.0 h1:zRh22SR7o4K35SoNqouS9J/TKHTyU2QWaj5ldehyXtA= +github.com/consensys/gnark-crypto v0.10.0/go.mod h1:Iq/P3HHl0ElSjsg2E1gsMwhAyxnxoKK5nVyZKd+/KhU= github.com/coocood/freecache v1.2.3 h1:lcBwpZrwBZRZyLk/8EMyQVXRiFl663cCuMOrjCALeto= github.com/coocood/freecache v1.2.3/go.mod h1:RBUWa/Cy+OHdfTGFEhEuE1pMCMX51Ncizj7rthiQ3vk= github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= @@ -246,7 +174,7 @@ github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7 github.com/coreos/go-systemd v0.0.0-20190719114852-fd7a80b32e1f/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= -github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= +github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE= github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/cpuguy83/go-md2man/v2 v2.0.2 h1:p1EgwI/C7NhT0JmVkwCD2ZBK8j4aeHQX2pMHHBfMQ6w= github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= @@ -254,51 +182,30 @@ github.com/crate-crypto/go-kzg-4844 v0.3.0 h1:UBlWE0CgyFqqzTI+IFyCzA7A3Zw4iip6uz github.com/crate-crypto/go-kzg-4844 v0.3.0/go.mod h1:SBP7ikXEgDnUPONgm33HtuDZEDtWa3L4QtN1ocJSEQ4= github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= -github.com/cyberdelia/templates v0.0.0-20141128023046-ca7fffd4298c/go.mod h1:GyV+0YP4qX0UQ7r2MoYZ+AvYDp12OF5yg4q8rGnyNh4= github.com/d4l3k/messagediff v1.2.1 h1:ZcAIMYsUg0EAp9X+tt8/enBE/Q8Yd5kzPynLyKptt9U= -github.com/dave/jennifer v1.2.0/go.mod h1:fIb+770HOpJ2fmN9EPPKOqm1vMGhB+TwXKMZhrIygKg= -github.com/davecgh/go-spew v0.0.0-20171005155431-ecdeabc65495/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/deckarep/golang-set v0.0.0-20180603214616-504e848d77ea/go.mod h1:93vsz/8Wt4joVM7c2AVqh+YRMiUSc14yDtF28KmMOgQ= -github.com/deckarep/golang-set v1.7.1/go.mod h1:93vsz/8Wt4joVM7c2AVqh+YRMiUSc14yDtF28KmMOgQ= -github.com/deckarep/golang-set/v2 v2.1.0 h1:g47V4Or+DUdzbs8FxCCmgb6VYd+ptPAngjM6dtGktsI= -github.com/deckarep/golang-set/v2 v2.1.0/go.mod h1:VAky9rY/yGXJOLEDv3OMci+7wtDpOF4IN+y82NBOac4= -github.com/decred/dcrd/crypto/blake256 v1.0.0 h1:/8DMNYp9SGi5f0w7uCm6d6M4OU2rGFK09Y2A4Xv7EE0= -github.com/decred/dcrd/dcrec/secp256k1/v4 v4.1.0 h1:HbphB4TFFXpv7MNrT52FGrrgVXF1owhMVTHFZIlnvd4= -github.com/decred/dcrd/dcrec/secp256k1/v4 v4.1.0/go.mod h1:DZGJHZMqrU4JJqFAWUS2UO1+lbSKsdiOoYi9Zzey7Fc= -github.com/decred/dcrd/lru v1.0.0/go.mod h1:mxKOwFd7lFjN2GZYsiz/ecgqR6kkYAl+0pz0tEMk218= -github.com/deepmap/oapi-codegen v1.6.0/go.mod h1:ryDa9AgbELGeB+YEXE1dR53yAjHwFvE9iAUlWl9Al3M= -github.com/deepmap/oapi-codegen v1.8.2/go.mod h1:YLgSKSDv/bZQB7N4ws6luhozi3cEdRktEqrX88CvjIw= +github.com/deckarep/golang-set/v2 v2.3.1 h1:vjmkvJt/IV27WXPyYQpAh4bRyWJc5Y435D17XQ9QU5A= +github.com/deckarep/golang-set/v2 v2.3.1/go.mod h1:VAky9rY/yGXJOLEDv3OMci+7wtDpOF4IN+y82NBOac4= +github.com/decred/dcrd/crypto/blake256 v1.0.1 h1:7PltbUIQB7u/FfZ39+DGa/ShuMyJ5ilcvdfma9wOH6Y= +github.com/decred/dcrd/dcrec/secp256k1/v4 v4.2.0 h1:8UrgZ3GkP4i/CLijOJx79Yu+etlyjdBU4sfcs2WYQMs= +github.com/decred/dcrd/dcrec/secp256k1/v4 v4.2.0/go.mod h1:v57UDF4pDQJcEfFUCRop3lJL149eHGSe9Jvczhzjo/0= +github.com/dgraph-io/badger v1.6.0/go.mod h1:zwt7syl517jmP8s94KqSxTlM6IMsdhYy6psNgSztDR4= github.com/dgraph-io/ristretto v0.1.1 h1:6CWw5tJNgpegArSHpNHJKldNeq03FQCwYvfMVWajOK8= github.com/dgraph-io/ristretto v0.1.1/go.mod h1:S1GPSBCYCIhmVNfcth17y2zZtQT6wzkzgwUve0VDWWA= github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= -github.com/dgryski/go-bitstream v0.0.0-20180413035011-3522498ce2c8/go.mod h1:VMaSuZ+SZcx/wljOQKvp5srsbCiKDEb6K2wC4+PiBmQ= github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2 h1:tdlZCpZ/P9DhczCTSixgIKmwPv6+wP5DGjqLYw5SUiA= github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78= github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc= github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= -github.com/dlclark/regexp2 v1.2.0/go.mod h1:2pZnwuY/m+8K6iRw6wQdMtk+rH5tNGR1i55kozfMjCc= -github.com/dlclark/regexp2 v1.4.1-0.20201116162257-a2a8dda75c91/go.mod h1:2pZnwuY/m+8K6iRw6wQdMtk+rH5tNGR1i55kozfMjCc= -github.com/docker/docker v1.4.2-0.20180625184442-8e610b2b55bf/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= github.com/donovanhide/eventsource v0.0.0-20210830082556-c59027999da0 h1:C7t6eeMaEQVy6e8CarIhscYQlNmw5e3G36y7l7Y21Ao= github.com/donovanhide/eventsource v0.0.0-20210830082556-c59027999da0/go.mod h1:56wL82FO0bfMU5RvfXoIwSOP2ggqqxT+tAfNEIyxuHw= -github.com/dop251/goja v0.0.0-20200721192441-a695b0cdd498/go.mod h1:Mw6PkjjMXWbTj+nnj4s3QPXq1jaT0s5pC0iFD4+BOAA= -github.com/dop251/goja v0.0.0-20211011172007-d99e4b8cbf48/go.mod h1:R9ET47fwRVRPZnOGvHxxhuZcbrMCuiqOz3Rlrh4KSnk= -github.com/dop251/goja_nodejs v0.0.0-20210225215109-d91c329300e7/go.mod h1:hn7BA7c8pLvoGndExHudxTDKZ84Pyvv+90pbBjbTz0Y= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= -github.com/dvyukov/go-fuzz v0.0.0-20200318091601-be3528f3a813/go.mod h1:11Gm+ccJnvAhCNLlf5+cS9KjtbaD5I5zaZpFMsTHWTw= -github.com/eapache/go-resiliency v1.2.0/go.mod h1:kFI+JgMyC7bLPUVY133qvEBtVayf5mFgVsvEsIPBvNs= -github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21/go.mod h1:+020luEh2TKB4/GOp8oxxtq0Daoen/Cii55CzbTV6DU= -github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I= -github.com/eclipse/paho.mqtt.golang v1.2.0/go.mod h1:H9keYFcgq3Qr5OUJm/JZI/i6U7joQ8SYLhZwfeOo6Ts= -github.com/edsrzf/mmap-go v0.0.0-20160512033002-935e0e8a636c/go.mod h1:YO35OhQPt3KJa3ryjFM5Bs14WD66h8eGKpfaBNrHW5M= -github.com/edsrzf/mmap-go v1.0.0 h1:CEBF7HpRnUCSJgGUb5h1Gm7e3VkmVDrR8lvWVLtrOFw= -github.com/edsrzf/mmap-go v1.0.0/go.mod h1:YO35OhQPt3KJa3ryjFM5Bs14WD66h8eGKpfaBNrHW5M= +github.com/eknkc/amber v0.0.0-20171010120322-cdade1c07385/go.mod h1:0vRUJqYpeSZifjYj7uP3BG/gKcuzL9xWVV/Y+cK33KM= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= @@ -309,11 +216,11 @@ github.com/envoyproxy/go-control-plane v0.10.2-0.20220325020618-49ff273808a1 h1: github.com/envoyproxy/go-control-plane v0.10.2-0.20220325020618-49ff273808a1/go.mod h1:KJwIaB5Mv44NWtYuAOFCVOjcI94vtpEz2JU/D2v6IjE= github.com/envoyproxy/protoc-gen-validate v0.1.0 h1:EQciDnbrYxy13PgWoY8AqoxGiPrpgBZ1R8UNe3ddc+A= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= -github.com/ethereum/go-ethereum v1.9.23/go.mod h1:JIfVb6esrqALTExdz9hRYvrP0xBDf6wCncIu1hNwHpM= -github.com/ethereum/go-ethereum v1.10.4/go.mod h1:nEE0TP5MtxGzOMd7egIrbPJMQBnhVU3ELNxhBglIzhg= -github.com/ethereum/go-ethereum v1.10.12/go.mod h1:W3yfrFyL9C1pHcwY5hmRHVDaorTiQxhYBkKyu5mEDHw= -github.com/ethereum/go-ethereum v1.11.3 h1:uuBkYUJW9aY5JYi3+sqLHz+XWyo5fmn/ab9XcbtVDTU= -github.com/ethereum/go-ethereum v1.11.3/go.mod h1:rBUvAl5cdVrAei9q5lgOU7RSEuPJk1nlBDnS/YSoKQE= +github.com/etcd-io/bbolt v1.3.3/go.mod h1:ZF2nL25h33cCyBtcyWeZ2/I3HQOfTP+0PIEvHjkjCrw= +github.com/ethereum/c-kzg-4844 v0.3.1 h1:sR65+68+WdnMKxseNWxSJuAv2tsUrihTpVBTfM/U5Zg= +github.com/ethereum/c-kzg-4844 v0.3.1/go.mod h1:VewdlzQmpT5QSrVhbBuGoCdFJkpaJlO1aQputP83wc0= +github.com/ethereum/go-ethereum v1.12.2 h1:eGHJ4ij7oyVqUQn48LBz3B7pvQ8sV0wGJiIE6gDq/6Y= +github.com/ethereum/go-ethereum v1.12.2/go.mod h1:1cRAEV+rp/xX0zraSCBnu9Py3HQ+geRMj3HdR+k0wfI= github.com/evanw/esbuild v0.8.23 h1:eRRG1fNtQ9KPG3lM62EUYagLVMSuxSTBEgukqY0et3w= github.com/evanw/esbuild v0.8.23/go.mod h1:y2AFBAGVelPqPodpdtxWWqe6n2jYf5FrsJbligmRmuw= github.com/facebookgo/ensure v0.0.0-20160127193407-b4ab57deab51 h1:0JZ+dUmQeA8IIVUMzysrX4/AKuQwWhV2dYQuPZdvdSQ= @@ -322,34 +229,25 @@ github.com/facebookgo/stack v0.0.0-20160209184415-751773369052 h1:JWuenKqqX8nojt github.com/facebookgo/stack v0.0.0-20160209184415-751773369052/go.mod h1:UbMTZqLaRiH3MsBH8va0n7s1pQYcu3uTb8G4tygF4Zg= github.com/facebookgo/subset v0.0.0-20150612182917-8dac2c3c4870 h1:E2s37DuLxFhQDg5gKsWoLBOB0n+ZW8s599zru8FJ2/Y= github.com/facebookgo/subset v0.0.0-20150612182917-8dac2c3c4870/go.mod h1:5tD+neXqOorC30/tWg0LCSkrqj/AR6gu8yY8/fpw1q0= -github.com/fatih/color v1.3.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= +github.com/fasthttp-contrib/websocket v0.0.0-20160511215533-1f3b11f56072/go.mod h1:duJ4Jxv5lDcvg4QuQr0oowTf7dz4/CR8NtyCooz9HL8= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/fatih/color v1.10.0/go.mod h1:ELkj/draVOlAH/xkhN6mQ50Qd0MPOk5AAr3maGEBuJM= github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk= github.com/fatih/color v1.14.1 h1:qfhVLaG5s+nCROl1zJsZRxFeYrHLqWroPOQ8BWiNb4w= github.com/fatih/color v1.14.1/go.mod h1:2oHN61fhTpgcxD3TSWCgKDiH1+x4OiDVVGH8WlgGZGg= -github.com/ferranbt/fastssz v0.0.0-20210120143747-11b9eff30ea9/go.mod h1:DyEu2iuLBnb/T51BlsiO3yLYdJC6UbGMrIkqK1KmQxM= -github.com/ferranbt/fastssz v0.0.0-20210905181407-59cf6761a7d5/go.mod h1:S8yiDeAXy8f88W4Ul+0dBMPx49S05byYbmZD6Uv94K4= +github.com/fatih/structs v1.1.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M= github.com/ferranbt/fastssz v0.1.2/go.mod h1:X5UPrE2u1UJjxHA8X54u04SBwdAQjG2sFtWs39YxyWs= github.com/ferranbt/fastssz v0.1.3 h1:ZI+z3JH05h4kgmFXdHuR1aWYsgrg7o+Fw7/NCzM16Mo= github.com/ferranbt/fastssz v0.1.3/go.mod h1:0Y9TEd/9XuFlh7mskMPfXiI2Dkw4Ddg9EyXt1W7MRvE= -github.com/fjl/memsize v0.0.0-20180418122429-ca190fb6ffbc/go.mod h1:VvhXpOYNQvB+uIk2RvXzuaQtkQJzzIx6lSBe1xv7hi0= github.com/fjl/memsize v0.0.0-20190710130421-bcb5799ab5e5 h1:FtmdgXiUlNeRsoNMFlKLDt+S+6hbjVMEW6RGQ7aUf7c= -github.com/fjl/memsize v0.0.0-20190710130421-bcb5799ab5e5/go.mod h1:VvhXpOYNQvB+uIk2RvXzuaQtkQJzzIx6lSBe1xv7hi0= -github.com/fogleman/gg v1.2.1-0.20190220221249-0403632d5b90/go.mod h1:R/bRT+9gY/C5z7JzPU0zXsXHKM4/ayA+zqcVNZzPa1k= -github.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g= -github.com/frankban/quicktest v1.7.2/go.mod h1:jaStnuzAqU1AJdCO0l53JDCJrVDKcS03DbaAcR7Ks/o= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= github.com/fsnotify/fsnotify v1.5.4/go.mod h1:OVB6XrOHzAwXMpEM7uPOzcehqUV2UqJxmVXmkdnm1bU= github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY= github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw= -github.com/garyburd/redigo v1.6.0/go.mod h1:NR3MbYisc3/PwhQ00EMzDiPmrwpPxAn5GI05/YaO1SY= -github.com/gballet/go-libpcsclite v0.0.0-20190607065134-2772fd86a8ff/go.mod h1:x7DCsMOv1taUwEWCzT4cmDeAkigA5/QCwUodaVOe8Ww= +github.com/gavv/httpexpect v2.0.0+incompatible/go.mod h1:x+9tiU1YnrOvnB725RkpoLv1M62hOWzwo5OXotisrKc= github.com/gballet/go-libpcsclite v0.0.0-20191108122812-4678299bea08 h1:f6D9Hr8xV8uYKlyuj8XIruxlh9WjVjdh1gIicAS7ays= -github.com/gballet/go-libpcsclite v0.0.0-20191108122812-4678299bea08/go.mod h1:x7DCsMOv1taUwEWCzT4cmDeAkigA5/QCwUodaVOe8Ww= -github.com/getkin/kin-openapi v0.53.0/go.mod h1:7Yn5whZr5kJi6t+kShccXS8ae1APpYTW6yheSwk8Yi4= -github.com/getkin/kin-openapi v0.61.0/go.mod h1:7Yn5whZr5kJi6t+kShccXS8ae1APpYTW6yheSwk8Yi4= +github.com/getsentry/sentry-go v0.12.0/go.mod h1:NSap0JBYWzHND8oMbyi0+XZhUalc1TBdRL1M71JZW2c= github.com/getsentry/sentry-go v0.18.0 h1:MtBW5H9QgdcJabtZcuJG80BMOwaBpkRDZkxRkNC1sN0= github.com/getsentry/sentry-go v0.18.0/go.mod h1:Kgon4Mby+FJ7ZWHFUAZgVaIa8sxHtnRJRLTXZr51aKQ= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= @@ -357,12 +255,12 @@ github.com/gin-contrib/sse v0.0.0-20190301062529-5545eab6dad3/go.mod h1:VJ0WA2NB github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm+fLHvGI= github.com/gin-gonic/gin v1.4.0/go.mod h1:OW2EZn3DO8Ln9oIKOvM++LBO+5UPHJJDH72/q/3rZdM= github.com/gin-gonic/gin v1.5.0/go.mod h1:Nd6IXA8m5kNZdNEHMBd93KT+mdY3+bewLgRvmCsR2Do= -github.com/glycerine/go-unsnap-stream v0.0.0-20180323001048-9f0cb55181dd/go.mod h1:/20jfyN9Y5QPEAprSgKAUr+glWDY39ZiUEAYOEv5dsE= -github.com/glycerine/goconvey v0.0.0-20190410193231-58a59202ab31/go.mod h1:Ogl1Tioa0aV7gstGFO7KhffUsb9M4ydbEbbxpcEDc24= +github.com/go-check/check v0.0.0-20180628173108-788fd7840127/go.mod h1:9ES+weclKsC9YodN5RgxqK/VD9HM9JsCSh7rNhMZE98= github.com/go-chi/chi v4.0.0+incompatible/go.mod h1:eB3wogJHnLi3x/kFX2A+IbTBlXxmMeXJVKy9tTv1XzQ= github.com/go-chi/chi v4.0.2+incompatible h1:maB6vn6FqCxrpz4FqWdh4+lwpyZIQS7YEAUcHlgXVRs= github.com/go-chi/chi v4.0.2+incompatible/go.mod h1:eB3wogJHnLi3x/kFX2A+IbTBlXxmMeXJVKy9tTv1XzQ= -github.com/go-chi/chi/v5 v5.0.0/go.mod h1:BBug9lr0cqtdAhsu6R4AAdvufI0/XBzAQSsUqJpoZOs= +github.com/go-errors/errors v1.0.1/go.mod h1:f4zRHt4oKfwPJE5k8C9vpYG+aDHdBFUsgrm6/TyX73Q= +github.com/go-errors/errors v1.4.2 h1:J6MZopCL4uSllY1OfXM374weqZFFItUbrImctkmUxIA= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= @@ -373,10 +271,7 @@ github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9 github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= github.com/go-logr/logr v1.2.3 h1:2DntVwHkVopvECVRSlL5PSo9eG+cAkDCuckLubN+rq0= -github.com/go-ole/go-ole v1.2.1/go.mod h1:7FAglXiTm7HKlQRDeOQ6ZNUHidzCWXuZWq/1dTyBNF8= -github.com/go-ole/go-ole v1.2.4/go.mod h1:XCwSNxSkXRo4vlyPy93sltvi/qJq0jqQhjqQNIwKuxM= -github.com/go-ole/go-ole v1.2.5/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= -github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY= +github.com/go-martini/martini v0.0.0-20170121215854-22fa46961aab/go.mod h1:/P9AEU963A2AYjv4d1V5eVL1CQbEJq6aCNHDDjibzu8= github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= github.com/go-ole/go-ole v1.3.0 h1:Dt6ye7+vXGIKZ7Xtk4s6/xVdGDQynvom7xCFEdWr6uE= github.com/go-ole/go-ole v1.3.0/go.mod h1:5LS6F96DhAwUc7C+1HLexzMXY1xGRSryjyPPKW6zv78= @@ -402,10 +297,7 @@ github.com/go-playground/validator/v10 v10.4.1/go.mod h1:nlOn6nFhuKACm19sB/8EGNn github.com/go-playground/validator/v10 v10.11.1 h1:prmOlTVv+YjZjmRmNSF3VmspqJIxJWXmqUsHwfTRRkQ= github.com/go-redis/redis/v8 v8.11.5 h1:AcZZR7igkdvfVmQTPnu9WE37LRrO/YrBH5zWyjDC0oI= github.com/go-redis/redis/v8 v8.11.5/go.mod h1:gREzHqY1hg6oD9ngVRbLStwAWKhA0FEgq8Jd4h5lpwo= -github.com/go-sourcemap/sourcemap v2.1.2+incompatible/go.mod h1:F8jJfvm2KbVjc5NqelyYJmf/v5J0dwNLS2mL4sNA1Jg= -github.com/go-sourcemap/sourcemap v2.1.3+incompatible/go.mod h1:F8jJfvm2KbVjc5NqelyYJmf/v5J0dwNLS2mL4sNA1Jg= github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= -github.com/go-sql-driver/mysql v1.4.1/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= github.com/go-sql-driver/mysql v1.7.0 h1:ueSltNNllEqE3qcWBTD0iQd3IpL/6U+mJxLkazJ7YPc= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/go-stack/stack v1.8.1 h1:ntEHSVwIt7PNXNpgPmVfMrNhLtgjlmnZha2kOpuRiDw= @@ -426,7 +318,7 @@ github.com/goccy/go-yaml v1.10.0/go.mod h1:h/18Lr6oSQ3mvmqFoWmQ47KChOgpfHpTyIHl3 github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/gofrs/flock v0.8.0/go.mod h1:F1TvTiK9OcQqauNUHlbJvyl9Qa1QvF/gOUDKA14jxHU= github.com/gofrs/flock v0.8.1 h1:+gYjHKf32LDeiEEFhQaotPbLuUXjY5ZqxKgXy7n59aw= -github.com/gofrs/uuid v3.3.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= +github.com/gofrs/flock v0.8.1/go.mod h1:F1TvTiK9OcQqauNUHlbJvyl9Qa1QvF/gOUDKA14jxHU= github.com/gofrs/uuid v4.0.0+incompatible h1:1SD/1F5pU8p29ybwgQSwpQk+mwdRrXCYuPhW6m+TnJw= github.com/gofrs/uuid v4.0.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= github.com/gogo/googleapis v0.0.0-20180223154316-0cd9801be74a/go.mod h1:gf4bu3Q80BeJ6H1S1vYPm8/ELATdvryBaNFGgqEef3s= @@ -434,14 +326,12 @@ github.com/gogo/googleapis v1.4.1/go.mod h1:2lpHqI5OcWCtVElxXnPt+s8oJvMpySlOyM6x github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.2.0/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= -github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/gogo/status v1.1.0/go.mod h1:BFv9nrluPLmrS0EmGVvLaPNmRosr9KapBYd5/hpY1WM= github.com/golang-jwt/jwt v3.2.2+incompatible h1:IfV12K8xAKAnZqdXVzCZ+TOjboZ2keLg81eXfW3O+oY= github.com/golang-jwt/jwt v3.2.2+incompatible/go.mod h1:8pz2t5EyA70fFQQSrl6XZXzqecmYZeUEB8OUGHkxJ+I= github.com/golang-jwt/jwt/v4 v4.3.0 h1:kHL1vqdqWNfATmA0FNMdmZNMyZI1U6O31X4rlIPoBog= -github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0/go.mod h1:E/TSTwGwJL78qG/PmXZO1EjYhfJinVAhrmmHX6Z8B9k= -github.com/golang/geo v0.0.0-20190916061304-5b978397cfec/go.mod h1:QZ0nwyI2jOfgRAoBvP+ab5aRr7c9x7lhGEJrKvBwjWI= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/glog v1.0.0 h1:nfP3RFugxnNRyKgeWd4oI1nYvXpxrx8ck8ZrcizshdQ= github.com/golang/glog v1.0.0/go.mod h1:EWib/APOK0SL3dFbYqvxE3UYd8E6s1ouQ7iEp/0LWV4= @@ -458,7 +348,6 @@ github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4= -github.com/golang/mock v1.5.0/go.mod h1:CWnOUgYIOo4TcNZ0wHX3YZCqsaM1I1Jvs6v3mP3KVu8= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= @@ -474,22 +363,18 @@ github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QD github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= -github.com/golang/protobuf v1.5.1/go.mod h1:DopwsBzvsk0Fs44TXzsVbJyPhcCPeIwnvohx4u74HPM= github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= -github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= -github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= -github.com/golang/snappy v0.0.2-0.20200707131729-196ae77b8a26/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= -github.com/golang/snappy v0.0.2/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= -github.com/golangci/lint-1 v0.0.0-20181222135242-d2cdd8c08219/go.mod h1:/X8TswGSh1pIozq4ZwCfxS0WA5JGXguxk94ar/4c87Y= +github.com/golang/snappy v0.0.5-0.20220116011046-fa5810519dcb h1:PBC98N2aIaM3XXiurYmW7fx4GZkL8feAMVq7nEjURHk= +github.com/golang/snappy v0.0.5-0.20220116011046-fa5810519dcb/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/gomodule/redigo v1.7.1-0.20190724094224-574c33c3df38/go.mod h1:B4C85qUVwatsJoIUNIfCRsp7qO0iAmpGFZ4EELWSbC4= github.com/gomodule/redigo v1.8.0 h1:OXfLQ/k8XpYF8f8sZKd2Df4SDyzbLeC35OsBsB11rYg= github.com/gomodule/redigo v1.8.0/go.mod h1:P9dn9mFrCBvWhGE1wpxx6fgq7BAeLBk+UUUzlpkBYO0= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.1.2 h1:xf4v41cLI2Z6FxbKm+8Bu+m8ifhj15JuZ9sa0jZCMUU= -github.com/google/flatbuffers v1.11.0/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= @@ -506,7 +391,6 @@ github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= -github.com/google/gofuzz v1.1.1-0.20200604201612-c04b05f3adfa/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= github.com/google/martian v2.1.0+incompatible h1:/CP5g8u/VJHijgedC/Legn3BAbAaWPgecwXBIDzw5no= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= @@ -522,16 +406,12 @@ github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hf github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20201023163331-3e6fc7fc9c4c/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20201203190320-1bf35d6f28c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/pprof v0.0.0-20210122040257-d980be63207e/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/pprof v0.0.0-20210226084205-cbba55b83ad5/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= -github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/subcommands v1.2.0/go.mod h1:ZjhPrFU+Olkh9WazFPsl27BQ4UPiG37m3yTrtFlrHVk= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/google/uuid v1.1.5/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/google/uuid v1.2.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= -github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.3.1 h1:KjJaJ9iWZ3jOFZIf1Lqf4laDRCasjl0BCmnEGxkdLb4= +github.com/google/uuid v1.3.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/enterprise-certificate-proxy v0.2.0 h1:y8Yozv7SZtlU//QXbezB6QkpuE6jMD2/gfzk4AftXjs= github.com/googleapis/enterprise-certificate-proxy v0.2.0/go.mod h1:8C0jb7/mgJe/9KK8Lm7X9ctZC2t60YyIpYEI16jx0Qg= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= @@ -548,12 +428,10 @@ github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI= github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= github.com/gorilla/securecookie v1.1.1 h1:miw7JPhV+b/lAHSXz4qd/nN9jRiAFV5FwjeKyCS8BvQ= github.com/gorilla/securecookie v1.1.1/go.mod h1:ra0sb63/xPlUeL+yeDciTfxMRAA+MP+HVt/4epWDjd4= -github.com/gorilla/websocket v1.4.1-0.20190629185528-ae1634f6a989/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= +github.com/gorilla/websocket v1.4.1/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/gorilla/websocket v1.5.0 h1:PPwGk2jz7EePpoHN/+ClbZu8SPxiqlu12wZP/3sWmnc= github.com/gorilla/websocket v1.5.0/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= -github.com/graph-gophers/graphql-go v0.0.0-20191115155744-f33e81362277/go.mod h1:9CQHMSxwO4MprSdzoIEobiHpoLtHm77vfxsvsIN5Vuc= -github.com/graph-gophers/graphql-go v0.0.0-20201113091052-beb923fada29/go.mod h1:9CQHMSxwO4MprSdzoIEobiHpoLtHm77vfxsvsIN5Vuc= github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= @@ -562,7 +440,6 @@ github.com/hashicorp/consul/api v1.1.0/go.mod h1:VmuI/Lkw1nC05EYQWNKwWGbkg+FbDBt github.com/hashicorp/consul/sdk v0.1.1/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8= github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/go-bexpr v0.1.10 h1:9kuI5PFotCboP3dkDYFr/wi0gg0QVbSNz5oFRpxn4uE= -github.com/hashicorp/go-bexpr v0.1.10/go.mod h1:oxlubA2vC/gFVfX1A6JGp7ls7uCDlfJn732ehYYg+g0= github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM= @@ -572,61 +449,43 @@ github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerX github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4= github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= -github.com/hashicorp/go-uuid v1.0.2/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= +github.com/hashicorp/go-version v1.2.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= github.com/hashicorp/go-version v1.6.0 h1:feTTfFNnjP967rlCxM/I9g701jU+RN74YKx2mOkIeek= github.com/hashicorp/go-version v1.6.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= -github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= github.com/hashicorp/golang-lru v0.5.5-0.20210104140557-80c98217689d h1:dg1dEPuWpEqDnvIw251EVy4zlP8gWbsGj4BsUKCRpYs= github.com/hashicorp/golang-lru v0.5.5-0.20210104140557-80c98217689d/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= -github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64= github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0mNTz8vQ= github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I= github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc= -github.com/herumi/bls-eth-go-binary v0.0.0-20200522010937-01d282b5380b/go.mod h1:luAnRm3OsMQeokhGzpYmc0ZKwawY7o87PUEP11Z7r7U= -github.com/herumi/bls-eth-go-binary v0.0.0-20210208203315-e81c3e745d31/go.mod h1:luAnRm3OsMQeokhGzpYmc0ZKwawY7o87PUEP11Z7r7U= -github.com/herumi/bls-eth-go-binary v0.0.0-20210917013441-d37c07cfda4e/go.mod h1:luAnRm3OsMQeokhGzpYmc0ZKwawY7o87PUEP11Z7r7U= -github.com/herumi/bls-eth-go-binary v1.28.1/go.mod h1:luAnRm3OsMQeokhGzpYmc0ZKwawY7o87PUEP11Z7r7U= github.com/herumi/bls-eth-go-binary v1.29.1 h1:XcNSHYTyNjEUVfWDCE2gtG5r95biTwd7MJUJF09LtSE= github.com/herumi/bls-eth-go-binary v1.29.1/go.mod h1:luAnRm3OsMQeokhGzpYmc0ZKwawY7o87PUEP11Z7r7U= github.com/holiman/billy v0.0.0-20230718173358-1c7e68d277a7 h1:3JQNjnMRil1yD0IfZKHF9GxxWKDJGj8I0IqOUol//sw= github.com/holiman/bloomfilter/v2 v2.0.3 h1:73e0e/V0tCydx14a0SCYS/EWCxgwLZ18CZcZKVu0fao= github.com/holiman/bloomfilter/v2 v2.0.3/go.mod h1:zpoh+gs7qcpqrHr3dB55AMiJwo0iURXE7ZOP9L9hSkA= -github.com/holiman/uint256 v1.1.1/go.mod h1:y4ga/t+u+Xwd7CpDgZESaRcWy0I7XMlTMA25ApIH5Jw= -github.com/holiman/uint256 v1.2.0 h1:gpSYcPLWGv4sG43I2mVLiDZCNDh/EpGjSk8tmtxitHM= github.com/holiman/uint256 v1.2.0/go.mod h1:y4ga/t+u+Xwd7CpDgZESaRcWy0I7XMlTMA25ApIH5Jw= +github.com/holiman/uint256 v1.2.3 h1:K8UWO1HUJpRMXBxbmaY1Y8IAMZC/RsKB+ArEnnK4l5o= +github.com/holiman/uint256 v1.2.3/go.mod h1:SC8Ryt4n+UBbPbIBKaG9zbbDlp4jOru9xFZmPzLUTxw= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= -github.com/huin/goupnp v1.0.0/go.mod h1:n9v9KO1tAxYH82qOn+UTIFQDmx5n1Zxd/ClZDMX7Bnc= -github.com/huin/goupnp v1.0.1-0.20210310174557-0ca763054c88/go.mod h1:nNs7wvRfN1eKaMknBydLNQU6146XQim8t4h+q90biWo= -github.com/huin/goupnp v1.0.2/go.mod h1:0dxJBVBHqTMjIUMkESDTNgOOx/Mw5wYIfyFmdzSamkM= github.com/huin/goupnp v1.0.3 h1:N8No57ls+MnjlB+JPiCVSOyy/ot7MJTqlo7rn+NYSqQ= -github.com/huin/goutil v0.0.0-20170803182201-1ca381bf3150/go.mod h1:PpLOETDnJ0o3iZrZfqZzyLl6l7F3c6L1oWn7OICBi6o= +github.com/hydrogen18/memlistener v0.0.0-20200120041712-dcc25e7acd91/go.mod h1:qEIFzExnS6016fRpRfxrExeVn2gbClQA99gQhnIcdhE= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/imdario/mergo v0.3.13 h1:lFzP57bqS/wsqKssCGmtLAb8A0wKjLGrve2q3PPVcBk= github.com/imdario/mergo v0.3.13/go.mod h1:4lJ1jqUDcsbIECGy0RUJAXNIhg+6ocWgb1ALK2O4oXg= -github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= +github.com/imkira/go-interpol v1.1.0/go.mod h1:z0h2/2T3XF8kyEPpRgJ3kmNv+C43p+I/CoI+jC3w2iA= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= -github.com/influxdata/flux v0.65.1/go.mod h1:J754/zds0vvpfwuq7Gc2wRdVwEodfpCFM7mYlOw2LqY= -github.com/influxdata/influxdb v1.2.3-0.20180221223340-01288bdb0883/go.mod h1:qZna6X/4elxqT3yI9iZYdZrWWdeFOOprn86kgg4+IzY= -github.com/influxdata/influxdb v1.8.3/go.mod h1:JugdFhsvvI8gadxOI6noqNeeBHvWNTbfYGtiAn+2jhI= -github.com/influxdata/influxdb-client-go/v2 v2.4.0/go.mod h1:vLNHdxTJkIf2mSLvGrpj8TCcISApPoXkaxP8g9uRlW8= -github.com/influxdata/influxdb1-client v0.0.0-20191209144304-8bf82d3c094d/go.mod h1:qj24IKcXYK6Iy9ceXlo3Tc+vtHo9lIhSX5JddghvEPo= -github.com/influxdata/influxql v1.1.1-0.20200828144457-65d3ef77d385/go.mod h1:gHp9y86a/pxhjJ+zMjNXiQAA197Xk9wLxaz+fGG+kWk= -github.com/influxdata/line-protocol v0.0.0-20180522152040-32c6aa80de5e/go.mod h1:4kt73NQhadE3daL3WhR5EJ/J2ocX0PZzwxQ0gXJ7oFE= -github.com/influxdata/line-protocol v0.0.0-20200327222509-2487e7298839/go.mod h1:xaLFMmpvUxqXtVkUJfg9QmT88cDaCJ3ZKgdZ78oO8Qo= -github.com/influxdata/line-protocol v0.0.0-20210311194329-9aa0e372d097/go.mod h1:xaLFMmpvUxqXtVkUJfg9QmT88cDaCJ3ZKgdZ78oO8Qo= -github.com/influxdata/promql/v2 v2.12.0/go.mod h1:fxOPu+DY0bqCTCECchSRtWfc+0X19ybifQhZoQNF5D8= -github.com/influxdata/roaring v0.4.13-0.20180809181101-fc520f41fab6/go.mod h1:bSgUQ7q5ZLSO+bKBGqJiCBGAl+9DxyW63zLTujjUlOE= -github.com/influxdata/tdigest v0.0.0-20181121200506-bf2b5ad3c0a9/go.mod h1:Js0mqiSBE6Ffsg94weZZ2c+v/ciT8QRHFOap7EKDrR0= -github.com/influxdata/usage-client v0.0.0-20160829180054-6d3895376368/go.mod h1:Wbbw6tYNvwa5dlB6304Sd+82Z3f7PmVZHVKU637d4po= -github.com/ipfs/go-cid v0.0.7/go.mod h1:6Ux9z5e+HpkQdckYoX1PG/6xqKspzlEIR5SDmgqgC/I= -github.com/ipfs/go-cid v0.3.2 h1:OGgOd+JCFM+y1DjWPmVH+2/4POtpDzwcr7VgnB7mZXc= -github.com/ipfs/go-cid v0.3.2/go.mod h1:gQ8pKqT/sUxGY+tIwy1RPpAojYu7jAyCp5Tz1svoupw= +github.com/ipfs/go-cid v0.4.1 h1:A/T3qGvxi4kpKWWcPC/PgbvDA2bjVLO7n4UeVwnbs/s= +github.com/ipfs/go-cid v0.4.1/go.mod h1:uQHwDeX4c6CtyrFwdqyhpNcxVewur1M7l7fNU7LKwZk= +github.com/iris-contrib/blackfriday v2.0.0+incompatible/go.mod h1:UzZ2bDEoaSGPbkg6SAB4att1aAwTmVIx/5gCVqeyUdI= +github.com/iris-contrib/go.uuid v2.0.0+incompatible/go.mod h1:iz2lgM/1UnEf1kP0L/+fafWORmlnuysV2EMP8MW+qe0= +github.com/iris-contrib/jade v1.1.3/go.mod h1:H/geBymxJhShH5kecoiOCSssPX7QWYH7UaeZTSWddIk= +github.com/iris-contrib/pongo2 v0.0.1/go.mod h1:Ssh+00+3GAZqSQb30AvBRNxBx7rf0GqwkjqxNd0u65g= +github.com/iris-contrib/schema v0.0.1/go.mod h1:urYA3uvUNG1TIIjOSCzHr9/LmbQo8LrOcOqfqxa4hXw= github.com/jackc/chunkreader v1.0.0/go.mod h1:RT6O25fNZIuasFJRyZ4R/Y2BbhasbmZXF9QQ7T3kePo= github.com/jackc/chunkreader/v2 v2.0.0/go.mod h1:odVSm741yZoC3dpHEUXIqA9tQRhFrgOHwnPIn9lDKlk= github.com/jackc/chunkreader/v2 v2.0.1 h1:i+RDz65UE+mmpjTfyz0MoVTnzeYxroil2G82ki7MGG8= @@ -676,42 +535,25 @@ github.com/jackc/puddle v0.0.0-20190608224051-11cab39313c9/go.mod h1:m4B5Dj62Y0f github.com/jackc/puddle v1.1.3/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= github.com/jackc/puddle v1.3.0 h1:eHK/5clGOatcjX3oWGBO/MpxpbHzSwud5EWTSCI+MX0= github.com/jackc/puddle v1.3.0/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= -github.com/jackpal/go-nat-pmp v1.0.2-0.20160603034137-1fa385a6f458/go.mod h1:QPH045xvCAeXUZOxsnwmrtiCoxIr9eob+4orBN1SBKc= github.com/jackpal/go-nat-pmp v1.0.2 h1:KzKSgb7qkJvOUTqYl9/Hg/me3pWgBmERKrTGD7BdWus= -github.com/jackpal/go-nat-pmp v1.0.2/go.mod h1:QPH045xvCAeXUZOxsnwmrtiCoxIr9eob+4orBN1SBKc= -github.com/jcmturner/gofork v1.0.0/go.mod h1:MK8+TM0La+2rjBD4jE12Kj1pCCxK7d2LK/UM3ncEo0o= -github.com/jedisct1/go-minisign v0.0.0-20190909160543-45766022959e/go.mod h1:G1CVv03EnqU1wYL2dFwXxW2An0az9JTl/ZsqXQeBlkU= -github.com/jessevdk/go-flags v0.0.0-20141203071132-1679536dcc89/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= -github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= github.com/jhump/protoreflect v1.8.1/go.mod h1:7GcYQDdMU/O/BBrl/cX6PNHpXh6cenjd8pneu5yW7Tg= -github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= -github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= -github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= github.com/jmoiron/sqlx v1.2.0 h1:41Ip0zITnmWNR/vHV+S4m+VoUivnWY5E4OJfLZjCJMA= github.com/jmoiron/sqlx v1.2.0/go.mod h1:1FEQNm3xlJgrMD+FBdI9+xvCksHtbpVBBw5dYhBSsks= github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= -github.com/jrick/logrotate v1.0.0/go.mod h1:LNinyqDIJnpAur+b8yyulnQw/wDuN1+BYKlTRt3OuAQ= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= -github.com/jsternberg/zap-logfmt v1.0.0/go.mod h1:uvPs/4X51zdkcm5jXl5SYoN+4RK21K8mysFmDaM/h+o= github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/juliangruber/go-intersect v1.1.0 h1:sc+y5dCjMMx0pAdYk/N6KBm00tD/f3tq+Iox7dYDUrY= github.com/juliangruber/go-intersect v1.1.0/go.mod h1:WMau+1kAmnlQnKiikekNJbtGtfmILU/mMU6H7AgKbWQ= -github.com/julienschmidt/httprouter v1.1.1-0.20170430222011-975b5c4c7c21/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= github.com/jung-kurt/gofpdf v1.0.3-0.20190309125859-24315acbbda5 h1:PJr+ZMXIecYc1Ey2zucXdR73SMBtgjPgwa31099IMv0= github.com/jung-kurt/gofpdf v1.0.3-0.20190309125859-24315acbbda5/go.mod h1:7Id9E/uU8ce6rXgefFLlgrJj/GYY22cpxn+r32jIOes= -github.com/jwilder/encoding v0.0.0-20170811194829-b4e1701a28ef/go.mod h1:Ct9fl0F6iIOGgxJ5npU/IUOhOhqlVrGjyIZc8/MagT0= -github.com/karalabe/usb v0.0.0-20190919080040-51dc0efba356/go.mod h1:Od972xHfMJowv7NGVDiWVxk2zxnWgjLlJzE+F4F7AGU= -github.com/karalabe/usb v0.0.0-20191104083709-911d15fe12a9/go.mod h1:Od972xHfMJowv7NGVDiWVxk2zxnWgjLlJzE+F4F7AGU= -github.com/karalabe/usb v0.0.0-20210518091819-4ea20957c210/go.mod h1:Od972xHfMJowv7NGVDiWVxk2zxnWgjLlJzE+F4F7AGU= -github.com/karalabe/usb v0.0.0-20211005121534-4c5740d64559/go.mod h1:Od972xHfMJowv7NGVDiWVxk2zxnWgjLlJzE+F4F7AGU= -github.com/karalabe/usb v0.0.2 h1:M6QQBNxF+CQ8OFvxrT90BA0qBOXymndZnk5q235mFc4= -github.com/karalabe/usb v0.0.2/go.mod h1:Od972xHfMJowv7NGVDiWVxk2zxnWgjLlJzE+F4F7AGU= +github.com/k0kubun/colorstring v0.0.0-20150214042306-9440f1994b88/go.mod h1:3w7q1U84EfirKl04SVQ/s7nPm1ZPhiXd34z40TNz36k= +github.com/kataras/golog v0.0.10/go.mod h1:yJ8YKCmyL+nWjERB90Qwn+bdyBZsaQwU3bTVFgkFIp8= github.com/kataras/i18n v0.0.5 h1:X9EQHxDhjpN0zh+Ry0PZvi0ODi9lf5mo4wiXWtOYhlY= github.com/kataras/i18n v0.0.5/go.mod h1:U0aKF7ANqGmFVs4WCexDTYGf8wg7Rb3mLJCmr/OuDoo= github.com/kataras/iris/v12 v12.1.8/go.mod h1:LMYy4VlP67TQ3Zgriz8RE2h2kMZV2SgMYbq3UhfoFmE= @@ -721,39 +563,29 @@ github.com/kataras/sitemap v0.0.5/go.mod h1:KY2eugMKiPwsJgx7+U103YZehfvNGOXURubc github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 h1:Z9n2FFNUXsshfwJMBgNA0RU6/i7WVaAegv3PtuIHPMs= github.com/kelseyhightower/envconfig v1.4.0 h1:Im6hONhd3pLkfDFsbRgu68RDNkGF1r3dvMUtDTo2cv8= github.com/kelseyhightower/envconfig v1.4.0/go.mod h1:cccZRl6mQpaq41TPp5QxidR+Sa3axMbJDNb//FQX6Gg= -github.com/kilic/bls12-381 v0.1.0 h1:encrdjqKMEvabVQ7qYOKu1OvhqpK4s47wDYtNiPtlp4= github.com/kilic/bls12-381 v0.1.0/go.mod h1:vDTTHJONJ6G+P2R74EhnyotQDTliQDnFEwhdmfzw1ig= github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= -github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= -github.com/kkdai/bstream v0.0.0-20161212061736-f391b8402d23/go.mod h1:J+Gs4SYgM6CZQHDETBtE9HaSEkGmuNXF86RwHhHUvq4= -github.com/klauspost/compress v1.4.0/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= -github.com/klauspost/compress v1.9.8/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= -github.com/klauspost/compress v1.10.1/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= +github.com/klauspost/compress v1.8.2/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= +github.com/klauspost/compress v1.9.7/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= github.com/klauspost/compress v1.11.7/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= github.com/klauspost/compress v1.16.0 h1:iULayQNOReoYUe+1qtKOqw9CwJv3aNQu8ivo7lw1HU4= github.com/klauspost/compress v1.16.0/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= -github.com/klauspost/cpuid v0.0.0-20170728055534-ae7887de9fa5/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= -github.com/klauspost/cpuid v1.2.3/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= +github.com/klauspost/cpuid v1.2.1/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= github.com/klauspost/cpuid/v2 v2.0.4/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= -github.com/klauspost/cpuid/v2 v2.0.6/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/klauspost/cpuid/v2 v2.1.2/go.mod h1:RVVoqg1df56z8g3pUjL/3lE5UfnlrJX8tyFgg4nqhuY= -github.com/klauspost/cpuid/v2 v2.2.4 h1:acbojRNwl3o09bUq+yDCtZFc1aiwaAAxtcn8YkZXnvk= -github.com/klauspost/cpuid/v2 v2.2.4/go.mod h1:RVVoqg1df56z8g3pUjL/3lE5UfnlrJX8tyFgg4nqhuY= -github.com/klauspost/crc32 v0.0.0-20161016154125-cb6bfca970f6/go.mod h1:+ZoRqAPRLkC4NPOvfYeR5KNOrY6TD+/sAC3HXPZgDYg= -github.com/klauspost/pgzip v1.0.2-0.20170402124221-0bf5dcad4ada/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs= +github.com/klauspost/cpuid/v2 v2.2.5 h1:0E5MSMDEoAulmXNFquVs//DdoomxaoTY1kUhbc/qbZg= +github.com/klauspost/cpuid/v2 v2.2.5/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= github.com/klauspost/pgzip v1.2.5/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs= -github.com/klauspost/reedsolomon v1.9.3/go.mod h1:CwCi+NUr9pqSVktrkN+Ondf06rkhYZ/pcNv7fu+8Un4= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= -github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= -github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= @@ -762,10 +594,9 @@ github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= -github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= -github.com/labstack/echo/v4 v4.2.1/go.mod h1:AA49e0DZ8kk5jTOOCKNuPR6oTnBS0dYiM4FW1e6jwpg= +github.com/labstack/echo/v4 v4.5.0/go.mod h1:czIriw4a0C1dFun+ObrXp7ok03xON0N1awStJ6ArI7Y= github.com/labstack/gommon v0.3.0/go.mod h1:MULnywXg0yavhxWKc+lOruYdAhDwPK9wf0OL7NoOu+k= -github.com/leanovate/gopter v0.2.9/go.mod h1:U2L/78B+KVFIx2VmW6onHJQzXtFb+p5y3y2Sh+Jxxv8= +github.com/leanovate/gopter v0.2.9 h1:fQjYxZaynp97ozCzfOyOuAGOU4aU/z37zf/tOujFk7c= github.com/leodido/go-urn v1.1.0/go.mod h1:+cyI34gQWZcE1eQU7NVgKkkzdXDQHr1dBMtdAPozLkw= github.com/leodido/go-urn v1.2.0/go.mod h1:+8+nEpDfqqsY+g338gtMEUOtuK+4dEMhiQEgxpxOKII= github.com/leodido/go-urn v1.2.1 h1:BqpAaACuzVSgi/VLzGZIobT2z4v53pjosyNd9Yv6n/w= @@ -775,10 +606,8 @@ github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/lib/pq v1.10.2/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= github.com/lib/pq v1.10.7 h1:p7ZhMD+KsSRozJr34udlUrhboJwWAgCg34+/ZZNvZZw= github.com/lib/pq v1.10.7/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= -github.com/libp2p/go-maddr-filter v0.1.0/go.mod h1:VzZhTXkMucEGGEOSKddrwGiOv0tUhgnKqNEmIAz/bPU= +github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= -github.com/magiconair/properties v1.8.5 h1:b6kJs+EmPFMYGkow9GiUyCyOvIwYetYJ3fSaWak/Gls= -github.com/magiconair/properties v1.8.5/go.mod h1:y3VJvCyxH9uVvJTWEGAELF3aiYNyPKd5NZ3oSwXrF60= github.com/mailgun/mailgun-go/v4 v4.1.3 h1:KLa5EZaOMMeyvY/lfAhWxv9ealB3mtUsMz0O9XmTtP0= github.com/mailgun/mailgun-go/v4 v4.1.3/go.mod h1:R9kHUQBptF4iSEjhriCQizplCDwrnDShy8w/iPiOfaM= github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= @@ -787,23 +616,16 @@ github.com/mailru/easyjson v0.7.0/go.mod h1:KAzv3t3aY1NaHWoQz1+4F1ccyAH66Jk7yos7 github.com/mailru/easyjson v0.7.6/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= -github.com/matryer/moq v0.0.0-20190312154309-6cfb0558e1bd/go.mod h1:9ELz6aaclSIGnZBoaSLZ3NAl1VTufbOrXBPvtcy6WiQ= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= -github.com/mattn/go-colorable v0.1.0/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcnceauSikq3lYCQ= github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= github.com/mattn/go-colorable v0.1.6/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= -github.com/mattn/go-colorable v0.1.7/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= github.com/mattn/go-colorable v0.1.8/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= github.com/mattn/go-colorable v0.1.9/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= github.com/mattn/go-colorable v0.1.11/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4= github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= -github.com/mattn/go-ieproxy v0.0.0-20190610004146-91bb50d98149/go.mod h1:31jz6HNzdxOmlERGGEc4v/dMssOfmp2p5bT/okiKFFc= -github.com/mattn/go-ieproxy v0.0.0-20190702010315-6dee0af9227d/go.mod h1:31jz6HNzdxOmlERGGEc4v/dMssOfmp2p5bT/okiKFFc= github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= -github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= -github.com/mattn/go-isatty v0.0.5-0.20180830101745-3fb116b82035/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.7/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= @@ -813,33 +635,22 @@ github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27k github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= github.com/mattn/go-isatty v0.0.17 h1:BTarxUcIeDqL27Mc+vyvdWYSL28zpIhv3RoTdsLMPng= github.com/mattn/go-isatty v0.0.17/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= -github.com/mattn/go-runewidth v0.0.3/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= -github.com/mattn/go-runewidth v0.0.4/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= -github.com/mattn/go-runewidth v0.0.7/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= -github.com/mattn/go-runewidth v0.0.13/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= github.com/mattn/go-runewidth v0.0.14 h1:+xnbZSEeDbOIg5/mE6JF0w6n9duR1l3/WmbinWVwUuU= github.com/mattn/go-runewidth v0.0.14/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= -github.com/mattn/go-sqlite3 v1.9.0 h1:pDRiWfl+++eC2FEFRy6jXmQlvp4Yh3z1MJKg4UeYM/4= github.com/mattn/go-sqlite3 v1.9.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= +github.com/mattn/go-sqlite3 v1.11.0 h1:LDdKkqtYlom37fkvqs8rMPFKAMe8+SgjbwZ6ex1/A/Q= github.com/mattn/go-sqlite3 v1.11.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= -github.com/mattn/go-tty v0.0.0-20180907095812-13ff1204f104/go.mod h1:XPvLUNfbS4fJH25nqRHfWLMa1ONC8Amw+mIA639KxkE= +github.com/mattn/goveralls v0.0.2/go.mod h1:8d1ZMHsd7fW6IRPKQh46F2WRpyib5/X4FOpevwGNQEw= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo= github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= github.com/mediocregopher/radix/v3 v3.4.2/go.mod h1:8FL3F6UQRXHXIBSPUs5h0RybMF8i4n7wVopoX3x7Bv8= github.com/microcosm-cc/bluemonday v1.0.2/go.mod h1:iVP4YcDBq+n/5fb23BhYFvIMq/leAFZyRl6bYmGDlGc= github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= -github.com/miekg/dns v1.1.43/go.mod h1:+evo5L0630/F6ca/Z9+GAqzhjGyn8/c+TBaOyfEl0V4= -github.com/miekg/dns v1.1.50 h1:DQUfb9uc6smULcREF09Uc+/Gd46YWqJd5DbpPE9xkcA= -github.com/miekg/dns v1.1.50/go.mod h1:e3IlAVfNqAllflbibAZEWOXOQ+Ynzk/dDozDxY7XnME= -github.com/minio/blake2b-simd v0.0.0-20160723061019-3f5f724cb5b1/go.mod h1:pD8RvIylQ358TN4wwqatJ8rNavkEINozVn9DtGI3dfQ= github.com/minio/highwayhash v1.0.2 h1:Aak5U0nElisjDCfPSG79Tgzkn2gl66NxOMspRrKnA/g= github.com/minio/highwayhash v1.0.2/go.mod h1:BQskDq+xkJ12lmlUUi7U0M5Swg3EWR+dLTk+kldvVxY= github.com/minio/sha256-simd v0.1.0/go.mod h1:2FMWW+8GMoPweT6+pI63m9YE3Lmw4J71hV56Chs1E/U= -github.com/minio/sha256-simd v0.1.1-0.20190913151208-6de447530771/go.mod h1:B5e1o+1/KgNmWrSQK08Y6Z1Vb5pwIktudl0J58iy0KM= -github.com/minio/sha256-simd v0.1.1/go.mod h1:B5e1o+1/KgNmWrSQK08Y6Z1Vb5pwIktudl0J58iy0KM= -github.com/minio/sha256-simd v1.0.0 h1:v1ta+49hkWZyvaKwrQB8elexRqm6Y0aMLjCNsrYxo6g= github.com/minio/sha256-simd v1.0.0/go.mod h1:OuYzVNI5vcoYIAmbIvHPl3N3jUzVedXbKy5RFepssQM= github.com/minio/sha256-simd v1.0.1 h1:6kaan5IFmwTNynnKKpDHe6FWHohJOHhCPchzK49dzMM= github.com/minio/sha256-simd v1.0.1/go.mod h1:Pz6AKMiUdngCLpeTL/RJY1M9rUuPMYujV5xJjtbRSN8= @@ -853,12 +664,12 @@ github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0Qu github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/mitchellh/mapstructure v1.3.2/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= -github.com/mitchellh/mapstructure v1.4.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= -github.com/mitchellh/mapstructure v1.4.2/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/mitchellh/pointerstructure v1.2.0 h1:O+i9nHnXS3l/9Wu7r4NrEdwA2VFTicjUEN1uBnDo34A= -github.com/mitchellh/pointerstructure v1.2.0/go.mod h1:BRAsLI5zgXmw97Lf6s25bs8ohIXc3tViBH44KcwB2g4= +github.com/mmcloughlin/addchain v0.4.0 h1:SobOdjm2xLj1KkXN5/n0xTIWyZA2+s99UCY1iPfkHRY= +github.com/mmcloughlin/addchain v0.4.0/go.mod h1:A86O+tHqZLMNO4w6ZZ4FlVQEadcoqkyU72HC5wJ4RlU= +github.com/mmcloughlin/profile v0.1.1/go.mod h1:IhHD7q1ooxgwTgjxQYkACGA77oFTDdFVejUS1/tS/qU= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -867,100 +678,65 @@ github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3Rllmb github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 h1:RWengNIwukTxcDr9M+97sNutRR1RKhG96O6jWumTTnw= github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826/go.mod h1:TaXosZuwdSHYgviHp1DAtfrULt5eUgsSMsZf+YrPgl8= -github.com/mr-tron/base58 v1.1.0/go.mod h1:xcD2VGqlgYjBdcBLw+TuYLr8afG+Hj8g2eTVqeSzSU8= -github.com/mr-tron/base58 v1.1.3/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjWI2mNwc= +github.com/moul/http2curl v1.0.0/go.mod h1:8UbvGypXm98wA/IqH45anm5Y2Z6ep6O31QGOAZ3H0fQ= github.com/mr-tron/base58 v1.2.0 h1:T/HDJBh4ZCPbU39/+c3rRvE0uKBQlU27+QI8LJ4t64o= github.com/mr-tron/base58 v1.2.0/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjWI2mNwc= -github.com/mschoch/smat v0.0.0-20160514031455-90eadee771ae/go.mod h1:qAyveg+e4CE+eKJXWVjKXM4ck2QobLqTDytGJbLLhJg= github.com/mssola/user_agent v0.5.2 h1:CZkTUahjL1+OcZ5zv3kZr8QiJ8jy2H08vZIEkBeRbxo= github.com/mssola/user_agent v0.5.2/go.mod h1:TTPno8LPY3wAIEKRpAtkdMT0f8SE24pLRGPahjCH4uw= -github.com/multiformats/go-base32 v0.0.3/go.mod h1:pLiuGC8y0QR3Ue4Zug5UzK9LjgbkL8NSQj0zQ5Nz/AA= github.com/multiformats/go-base32 v0.1.0 h1:pVx9xoSPqEIQG8o+UbAe7DNi51oej1NtK+aGkbLYxPE= github.com/multiformats/go-base32 v0.1.0/go.mod h1:Kj3tFY6zNr+ABYMqeUNeGvkIC/UYgtWibDcT0rExnbI= -github.com/multiformats/go-base36 v0.1.0/go.mod h1:kFGE83c6s80PklsHO9sRn2NCoffoRdUUOENyW/Vv6sM= github.com/multiformats/go-base36 v0.2.0 h1:lFsAbNOGeKtuKozrtBsAkSVhv1p9D0/qedU9rQyccr0= github.com/multiformats/go-base36 v0.2.0/go.mod h1:qvnKE++v+2MWCfePClUEjE78Z7P2a1UV0xHgWc0hkp4= -github.com/multiformats/go-multiaddr v0.2.2/go.mod h1:NtfXiOtHvghW9KojvtySjH5y0u0xW5UouOmQQrn6a3Y= -github.com/multiformats/go-multiaddr v0.3.2/go.mod h1:lCKNGP1EQ1eZ35Za2wlqnabm9xQkib3fyB+nZXHLag0= -github.com/multiformats/go-multibase v0.0.3/go.mod h1:5+1R4eQrT3PkYZ24C3W2Ue2tPwIdYQD509ZjSb5y9Oc= -github.com/multiformats/go-multibase v0.1.1 h1:3ASCDsuLX8+j4kx58qnJ4YFq/JWTJpCyDW27ztsVTOI= -github.com/multiformats/go-multibase v0.1.1/go.mod h1:ZEjHE+IsUrgp5mhlEAYjMtZwK1k4haNkcaPg9aoe1a8= -github.com/multiformats/go-multihash v0.0.13/go.mod h1:VdAWLKTwram9oKAatUcLxBNUjdtcVwxObEQBtRfuyjc= -github.com/multiformats/go-multihash v0.0.14/go.mod h1:VdAWLKTwram9oKAatUcLxBNUjdtcVwxObEQBtRfuyjc= -github.com/multiformats/go-multihash v0.0.15/go.mod h1:D6aZrWNLFTV/ynMpKsNtB40mJzmCl4jb1alC0OvHiHg= -github.com/multiformats/go-multihash v0.2.1 h1:aem8ZT0VA2nCHHk7bPJ1BjUbHNciqZC/d16Vve9l108= -github.com/multiformats/go-multihash v0.2.1/go.mod h1:WxoMcYG85AZVQUyRyo9s4wULvW5qrI9vb2Lt6evduFc= -github.com/multiformats/go-varint v0.0.5/go.mod h1:3Ls8CIEsrijN6+B7PbrXRPxHRPuXSrVKRY101jdMZYE= -github.com/multiformats/go-varint v0.0.6/go.mod h1:3Ls8CIEsrijN6+B7PbrXRPxHRPuXSrVKRY101jdMZYE= +github.com/multiformats/go-multibase v0.2.0 h1:isdYCVLvksgWlMW9OZRYJEa9pZETFivncJHmHnnd87g= +github.com/multiformats/go-multibase v0.2.0/go.mod h1:bFBZX4lKCA/2lyOFSAoKH5SS6oPyjtnzK/XTFDPkNuk= +github.com/multiformats/go-multihash v0.2.3 h1:7Lyc8XfX/IY2jWb/gI7JP+o7JEq9hOa7BFvVU9RSh+U= +github.com/multiformats/go-multihash v0.2.3/go.mod h1:dXgKXCXjBzdscBLk9JkjINiEsCKRVch90MdaGiKsvSM= github.com/multiformats/go-varint v0.0.7 h1:sWSGR+f/eu5ABZA2ZpYKBILXTTs9JWpdEM/nEGOHFS8= github.com/multiformats/go-varint v0.0.7/go.mod h1:r8PUYw/fD/SjBCiKOoDlGF6QawOELpZAu9eioSos/OU= github.com/mvdan/xurls v1.1.0 h1:OpuDelGQ1R1ueQ6sSryzi6P+1RtBpfQHM8fJwlE45ww= github.com/mvdan/xurls v1.1.0/go.mod h1:tQlNn3BED8bE/15hnSL2HLkDeLWpNPAwtw7wkEq44oU= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= -github.com/naoina/go-stringutil v0.1.0/go.mod h1:XJ2SJL9jCtBh+P9q5btrd/Ylo8XwT/h1USek5+NqSA0= -github.com/naoina/toml v0.1.2-0.20170918210437-9fafd6967416/go.mod h1:NBIhNtsFMo3G2szEBne+bO4gS192HuIYRqfvOWb4i1E= +github.com/nats-io/jwt v0.3.0/go.mod h1:fRYCDE99xlTsqUzISS1Bi75UBJ6ljOJQOAAu5VglpSg= +github.com/nats-io/nats.go v1.9.1/go.mod h1:ZjDU1L/7fJ09jvUSRVBR2e7+RnLiiIQyqyzEE/Zbp4w= +github.com/nats-io/nkeys v0.1.0/go.mod h1:xpnFELMwJABBLVhffcfd1MZx6VsNRFpEugbxziKVo7w= +github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= github.com/nishanths/predeclared v0.0.0-20200524104333-86fad755b4d3/go.mod h1:nt3d53pc1VYcphSCIaYAJtnPYnr3Zyn8fMq2wvPGPso= github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= -github.com/olekukonko/tablewriter v0.0.1/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo= -github.com/olekukonko/tablewriter v0.0.2-0.20190409134802-7e037d187b0c/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo= -github.com/olekukonko/tablewriter v0.0.4/go.mod h1:zq6QwlOf5SlnkVbMSr5EoBv3636FWnp+qbPhuoO21uA= github.com/olekukonko/tablewriter v0.0.5 h1:P2Ga83D34wi1o9J6Wh1mRuqd4mF/x/lgBS7N7AbDhec= github.com/olekukonko/tablewriter v0.0.5/go.mod h1:hPp6KlRPjbx+hW8ykQs1w3UBbZlj6HuIJcUGPhkA7kY= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= -github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= -github.com/onsi/ginkgo v1.10.1/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.10.3/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= -github.com/onsi/ginkgo v1.14.0/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY= github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vvnwo0= github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE= github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU= github.com/onsi/ginkgo/v2 v2.1.3/go.mod h1:vw5CSIxN1JObi/U8gcbwft7ZxR2dgaR70JSE3/PpL4c= -github.com/onsi/gomega v1.4.1/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= -github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= -github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= github.com/onsi/gomega v1.17.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAlGdZY= github.com/onsi/gomega v1.19.0 h1:4ieX6qQjPP/BfC3mpsAtIGGlxTWPeA3Inl/7DtXw1tw= github.com/onsi/gomega v1.19.0/go.mod h1:LY+I3pBVzYsTBU1AnDwOSxaYi9WoWiqgwooUqq9yPro= -github.com/openconfig/gnmi v0.0.0-20190823184014-89b2bf29312c/go.mod h1:t+O9It+LKzfOAhKTT5O0ehDix+MTqbtT0T9t+7zzOvc= -github.com/openconfig/reference v0.0.0-20190727015836-8dfd928c9696/go.mod h1:ym2A+zigScwkSEb/cVQB0/ZMpU3rqiH6X7WRRsxgOGw= -github.com/opentracing/opentracing-go v1.0.2/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= -github.com/opentracing/opentracing-go v1.0.3-0.20180606204148-bd9c31933947/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= -github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaRUnok+kx1WdO15EQc= github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ= -github.com/paulbellamy/ratecounter v0.2.0/go.mod h1:Hfx1hDpSGoqxkVVpBi/IlYD7kChlfo5C6hzIHwPqfFE= github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 h1:onHthvaw9LFnH4t2DcNVpwGmV9E1BkGknEliJkfwQj0= github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58/go.mod h1:DXv8WO4yhMYhSNPKjeNKa5WY9YCIEBRbNzFFPJbWO6Y= -github.com/pborman/uuid v0.0.0-20170112150404-1b00554d8222/go.mod h1:VyrYX9gd7irzKovcSS6BIIEwPRkP2Wm2m9ufcdFSJ34= -github.com/pborman/uuid v1.2.1 h1:+ZZIw58t/ozdjRaXh/3awHfmWRbzYxJoAdNJxe/3pvw= -github.com/pborman/uuid v1.2.1/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= -github.com/pelletier/go-toml v1.9.3 h1:zeC5b1GviRUyKYd6OJPvBU/mcVDVoL1OhT17FCt5dSQ= -github.com/pelletier/go-toml v1.9.3/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= -github.com/peterh/liner v1.0.1-0.20180619022028-8c1271fcf47f/go.mod h1:xIteQHvHuaLYG9IFj6mSxM0fCKrs34IrEQUhOYuGPHc= -github.com/peterh/liner v1.1.1-0.20190123174540-a2c9a5303de7/go.mod h1:CRroGNssyjTd/qIG2FyxByd2S8JEAZXBl4qUrZf8GS0= -github.com/peterh/liner v1.2.0/go.mod h1:CRroGNssyjTd/qIG2FyxByd2S8JEAZXBl4qUrZf8GS0= -github.com/peterh/liner v1.2.1/go.mod h1:CRroGNssyjTd/qIG2FyxByd2S8JEAZXBl4qUrZf8GS0= -github.com/philhofer/fwd v1.0.0/go.mod h1:gk3iGcWd9+svBvR0sR+KPcfE+RNWozjowpeBVG3ZVNU= github.com/phyber/negroni-gzip v0.0.0-20180113114010-ef6356a5d029 h1:d6HcSW4ZoNlUWrPyZtBwIu8yv4WAWIU3R/jorwVkFtQ= github.com/phyber/negroni-gzip v0.0.0-20180113114010-ef6356a5d029/go.mod h1:94RTq2fypdZCze25ZEZSjtbAQRT3cL/8EuRUqAZC/+w= -github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= -github.com/pierrec/lz4 v2.4.1+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= +github.com/pingcap/errors v0.11.4 h1:lFuQV/oaUMGcD2tqt+01ROSmJs75VG1ToEOkZIZ4nE4= +github.com/pingcap/errors v0.11.4/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8= +github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/profile v1.5.0/go.mod h1:qBsxPvzyUincmltOk6iyRVxHYg4adc0OFOv72ZdLa18= -github.com/pkg/sftp v1.10.1/go.mod h1:lYOWFsE0bwd1+KfKJaKeuokY15vzFx25BLbzYYoAxZI= -github.com/pkg/term v0.0.0-20180730021639-bffc007b7fd5/go.mod h1:eCbImbZ95eXtAUIbLAuAVnBnwf83mjf6QIVH8SHYwqQ= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI= @@ -971,7 +747,6 @@ github.com/prestonvanloon/go v1.1.7-0.20190722034630-4f2e55fcf87b/go.mod h1:KdQU github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso= github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= -github.com/prometheus/client_golang v1.4.1/go.mod h1:e9GMxYsXl05ICDXkRhurwBS4Q3OK1iX/F2sw+iXX5zU= github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M= github.com/prometheus/client_golang v1.11.0/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0= github.com/prometheus/client_golang v1.12.1/go.mod h1:3Z9XVyYiZYEO+YQWt3RD2R3jrbd179Rt297l4aS6nDY= @@ -986,8 +761,6 @@ github.com/prometheus/client_model v0.3.0/go.mod h1:LDGWKZIo7rky3hgvBe+caln+Dr3d github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= -github.com/prometheus/common v0.6.0/go.mod h1:eBmuwkDJBwy6iBfxCBob6t6dR6ENT/y+J+Zk0j9GMYc= -github.com/prometheus/common v0.9.1/go.mod h1:yhUN8i9wzaXS3w1O07YhxHEBxD+W35wd8bs7vj7HSQ4= github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo= github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc= github.com/prometheus/common v0.32.1/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls= @@ -996,33 +769,18 @@ github.com/prometheus/common v0.39.0/go.mod h1:6XBZ7lYdLCbkAVhwRsWTZn+IN5AB9F/NX github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= -github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A= -github.com/prometheus/procfs v0.0.10/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A= github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= github.com/prometheus/procfs v0.9.0 h1:wzCHvIvM5SxWqYvwgVL7yJY8Lz3PKn49KQtpgMYJfhI= github.com/prometheus/procfs v0.9.0/go.mod h1:+pB4zwohETzFnmlpe6yd2lSc+0/46IYZRB/chUwxUZY= -github.com/prometheus/tsdb v0.6.2-0.20190402121629-4f204dcbc150/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= -github.com/prometheus/tsdb v0.10.0/go.mod h1:oi49uRhEe9dPUTlS3JRZOwJuVi6tmh10QSgwXEyGCt4= -github.com/protolambda/bls12-381-util v0.0.0-20210720105258-a772f2aac13e h1:ugvwIKDzqL6ODJciRPMm+9xFQ5AlOYHeMpCOeEuP7LA= github.com/protolambda/bls12-381-util v0.0.0-20210720105258-a772f2aac13e/go.mod h1:MPZvj2Pr0N8/dXyTPS5REeg2sdLG7t8DRzC1rLv925w= -github.com/protolambda/eth2-val-tools v0.1.1 h1:IiiRtMO6ki6KRCJZuhLx46zeTEmRcufP8J0+7UcpBFk= -github.com/protolambda/eth2-val-tools v0.1.1/go.mod h1:CB/bzTq6ubTBmOIWk/Yga6e3zugcPYZEbiHG1pNHjaI= -github.com/protolambda/eth2-val-tools v0.1.2-0.20230223182759-4bf01453537a h1:d5tJf7+aZVqGnjM60W8QgHzAH88eKR1MjbVya2YuvDk= -github.com/protolambda/eth2-val-tools v0.1.2-0.20230223182759-4bf01453537a/go.mod h1:CB/bzTq6ubTBmOIWk/Yga6e3zugcPYZEbiHG1pNHjaI= -github.com/protolambda/go-keystorev4 v0.0.0-20211007151826-f20444f6d564 h1:yCXGkFjrZ8EggxW+Y7ueRZesNcBk0avLU0mVU/I2KtU= -github.com/protolambda/go-keystorev4 v0.0.0-20211007151826-f20444f6d564/go.mod h1:Xda3KO8+DMyWaTr+LwUUpVRTB5SdFzoKu0ivXNI6p1s= github.com/protolambda/messagediff v1.4.0/go.mod h1:LboJp0EwIbJsePYpzh5Op/9G1/4mIztMRYzzwR0dR2M= -github.com/protolambda/zrnt v0.12.4 h1:BJE8HSYx9wJnOOBr/QdYCE0fxUOfJpSKbSCcXGprm28= -github.com/protolambda/zrnt v0.12.4/go.mod h1:7G3u3+BgUjmQw75fQ3OYOJucujRrspBmshCx3OdRutc= github.com/protolambda/zrnt v0.30.0 h1:pHEn69ZgaDFGpLGGYG1oD7DvYI7RDirbMBPfbC+8p4g= github.com/protolambda/zrnt v0.30.0/go.mod h1:qcdX9CXFeVNCQK/q0nswpzhd+31RHMk2Ax/2lMsJ4Jw= github.com/protolambda/zssz v0.1.5 h1:7fjJjissZIIaa2QcvmhS/pZISMX21zVITt49sW1ouek= github.com/protolambda/zssz v0.1.5/go.mod h1:a4iwOX5FE7/JkKA+J/PH0Mjo9oXftN6P8NZyL28gpag= -github.com/protolambda/ztyp v0.1.0/go.mod h1:u9yT4ioIokwlHrOfiZ52ezHfKdza3phxhTJix01vGgU= -github.com/protolambda/ztyp v0.2.2 h1:rVcL3vBu9W/aV646zF6caLS/dyn9BN8NYiuJzicLNyY= github.com/protolambda/ztyp v0.2.2/go.mod h1:9bYgKGqg3wJqT9ac1gI2hnVb0STQq7p/1lapqrqY1dU= github.com/prysmaticlabs/fastssz v0.0.0-20221107182844-78142813af44 h1:c3p3UzV4vFA7xaCDphnDWOjpxcadrQ26l5b+ypsvyxo= github.com/prysmaticlabs/fastssz v0.0.0-20221107182844-78142813af44/go.mod h1:MA5zShstUwCQaE9faGHgCGvEWUbG87p4SAXINhmCkvg= @@ -1038,13 +796,10 @@ github.com/prysmaticlabs/grpc-gateway/v2 v2.3.1-0.20210702154020-550e1cd83ec1/go github.com/prysmaticlabs/protoc-gen-go-cast v0.0.0-20211014160335-757fae4f38c6 h1:+jhXLjEYVW4qU2z5SOxlxN+Hv/A9FDf0HpfDurfMEz0= github.com/r3labs/sse/v2 v2.7.4 h1:pvCMswPDlXd/ZUFx1dry0LbXJNHXwWPulLcUGYwClc0= github.com/r3labs/sse/v2 v2.7.4/go.mod h1:hUrYMKfu9WquG9MyI0r6TKiNH+6Sw/QPKm2YbNbU5g8= -github.com/rcrowley/go-metrics v0.0.0-20190826022208-cac0b30c2563/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE= -github.com/retailnext/hllpp v1.0.1-0.20180308014038-101a6d2f8b52/go.mod h1:RDpi1RftBQPUCDRw6SmxeaREsAaRKnOclghuzp/WRzc= github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= github.com/rivo/uniseg v0.4.3 h1:utMvzDsuh3suAEnhH0RdHmoPbU648o6CvXxTx4SBMOw= -github.com/rjeczalik/notify v0.9.1/go.mod h1:rKwnCoCGeuQnwBtTSPL9Dad03Vh2n40ePRrjvIXnJho= -github.com/rjeczalik/notify v0.9.2/go.mod h1:aErll2f0sUX9PXZnVNyeiObbmTlk5jnMoCa4QEjJeqM= +github.com/rivo/uniseg v0.4.3/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= github.com/rocket-pool/go-merkletree v1.0.1-0.20220406020931-c262d9b976dd h1:p9KuetSKB9nte9I/MkkiM3pwKFVQgqxxPTQ0y56Ff6s= github.com/rocket-pool/go-merkletree v1.0.1-0.20220406020931-c262d9b976dd/go.mod h1:UE9fof8P7iESVtLn1K9CTSkNRYVFHZHlf96RKbU33kA= github.com/rocket-pool/rocketpool-go v1.10.1-0.20230228020137-d5a680907dff h1:rOZevts77yp6t7J9xPRxhM1s4xyp4vbfqQJlFIWdDGE= @@ -1057,11 +812,9 @@ github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFR github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= github.com/rogpeppe/go-internal v1.8.1/go.mod h1:JeRgkft04UBgHMgCIwADu4Pn6Mtm5d4nPKWu0nJ5d+o= github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8= -github.com/rs/cors v0.0.0-20160617231935-a62a804a8a00/go.mod h1:gFx+x8UowdsKA9AchylcLynDq+nNFfI8FkUZdN/jGCU= -github.com/rs/cors v1.7.0/go.mod h1:gFx+x8UowdsKA9AchylcLynDq+nNFfI8FkUZdN/jGCU= +github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= github.com/rs/cors v1.8.0 h1:P2KMzcFwrPoSjkF1WLRPsp3UMLyql8L4v9hQpVeK5so= github.com/rs/cors v1.8.0/go.mod h1:EBwu+T5AvHOcXwvZIkQFjUN6s8Czyqw12GL/Y0tUyRM= -github.com/rs/xhandler v0.0.0-20160618193221-ed27b6fd6521/go.mod h1:RvLn4FgxWubrpZHtQLnOf6EwhN2hEMusxZOhcW9H3UQ= github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ= github.com/rs/xid v1.3.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= github.com/rs/zerolog v1.13.0/go.mod h1:YbFCdg8HfsridGWAh22vktObvhZbQsZXe4/zB0OKkWU= @@ -1077,14 +830,7 @@ github.com/ryanuber/columnize v2.1.0+incompatible/go.mod h1:sm1tb6uqfes/u+d4ooFo github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= github.com/schollz/closestmatch v2.1.0+incompatible/go.mod h1:RtP1ddjLong6gTkbtmuhtR2uUrrJOpYzYRvbcPAid+g= github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= -github.com/segmentio/kafka-go v0.1.0/go.mod h1:X6itGqS9L4jDletMsxZ7Dz+JFWxM6JHfPOCvTvk+EJo= -github.com/segmentio/kafka-go v0.2.0/go.mod h1:X6itGqS9L4jDletMsxZ7Dz+JFWxM6JHfPOCvTvk+EJo= github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= -github.com/shirou/gopsutil v2.20.5+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA= -github.com/shirou/gopsutil v2.20.9+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA= -github.com/shirou/gopsutil v3.21.4-0.20210419000835-c7a38de76ee5+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA= -github.com/shirou/gopsutil v3.21.5+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA= -github.com/shirou/gopsutil v3.21.7+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA= github.com/shirou/gopsutil v3.21.11+incompatible h1:+1+c1VGhc88SSonWP6foOcLhvnKlUeu/erjjvaPEYiI= github.com/shirou/gopsutil v3.21.11+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA= github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24/go.mod h1:M+9NzErvs504Cn4c5DxATwIqPbtswREoFCre64PpcG4= @@ -1096,7 +842,6 @@ github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPx github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= -github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/sirupsen/logrus v1.9.0 h1:trlNQbNUG3OdDrDil03MCb1H2o9nJ1x4/5LYw7byDE0= github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e h1:MRM5ITcdelLK2j1vwZ3Je0FKVCfqOLp5zO6trqMLYs0= @@ -1108,39 +853,20 @@ github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasO github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI= github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= -github.com/spf13/afero v1.6.0 h1:xoax2sJ2DT8S8xA2paPFjDCScCNeWsg75VG0DLRreiY= -github.com/spf13/afero v1.6.0/go.mod h1:Ai8FlHk4v/PARR026UzYexafAt9roJ7LcLMAmO6Z93I= github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= -github.com/spf13/cast v1.3.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= -github.com/spf13/cast v1.4.0 h1:WhlbjwB9EGCc8W5Rxdkus+wmH2ASRwwTJk6tgHKwdqQ= -github.com/spf13/cast v1.4.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= -github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= +github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU= github.com/spf13/cobra v1.0.1-0.20201006035406-b97b5ead31f7/go.mod h1:yk5b0mALVusDL5fMM6Rd1wgnoO5jUPhwsQ6LQAJTidQ= -github.com/spf13/cobra v1.1.3 h1:xghbfqPkxzxP3C/f3n5DdpAbdKLj4ZE4BWQI362l53M= -github.com/spf13/cobra v1.1.3/go.mod h1:pGADOWyqRD/YMrPZigI/zbliZ2wVD/23d+is3pSWzOo= -github.com/spf13/cobra v1.2.1 h1:+KmjbUw1hriSNMF55oPrkZcb27aECyrj8V2ytv7kWDw= -github.com/spf13/cobra v1.2.1/go.mod h1:ExllRjgxM/piMAM+3tAZvg8fsklGAf3tPfi+i8t68Nk= github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= -github.com/spf13/jwalterweatherman v1.1.0 h1:ue6voC5bR5F8YxI5S67j9i582FU4Qvo2bmqnqMYADFk= -github.com/spf13/jwalterweatherman v1.1.0/go.mod h1:aNWZUN0dPAAO/Ljvb5BEdw96iTZ0EXowPYD95IqWIGo= github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= -github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s= github.com/spf13/viper v1.7.0/go.mod h1:8WkrPz2fc9jxqZNCJI/76HCieCp4Q8HaLFoCha5qpdg= -github.com/spf13/viper v1.8.1 h1:Kq1fyeebqsBfbjZj4EL7gj2IO0mMaiyjYUWcUsl2O44= -github.com/spf13/viper v1.8.1/go.mod h1:o0Pch8wJ9BVSWGQMbra6iw0oQ5oktSIBaujf1rJH9Ns= -github.com/status-im/keycard-go v0.0.0-20190316090335-8537d3370df4/go.mod h1:RZLeN1LMWmRsyYjvAu+I6Dm9QmlDaIIt+Y+4Kd7Tp+Q= -github.com/status-im/keycard-go v0.0.0-20200402102358-957c09536969/go.mod h1:RZLeN1LMWmRsyYjvAu+I6Dm9QmlDaIIt+Y+4Kd7Tp+Q= github.com/status-im/keycard-go v0.2.0 h1:QDLFswOQu1r5jsycloeQh3bVU8n/NatHHaZobtDnDzA= -github.com/steakknife/bloomfilter v0.0.0-20180922174646-6819c0d2a570/go.mod h1:8OR4w3TdeIHIh1g6EMY5p0gVNOovcWC+1vpc7naMuAw= -github.com/steakknife/hamming v0.0.0-20180906055917-c99c65617cd3/go.mod h1:hpGUWaI9xL8pRQCTXQgocU38Qw1g0Us7n5PxxTwTCYU= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= -github.com/stretchr/testify v1.2.0/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= @@ -1154,7 +880,6 @@ github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= github.com/stripe/stripe-go/v72 v72.50.0 h1:oy+EsSKMrFS3zzayb8Ic+2LZ04Ux0vJ4990/7psaYsc= github.com/stripe/stripe-go/v72 v72.50.0/go.mod h1:QwqJQtduHubZht9mek5sds9CtQcKFdsykV9ZepRWwo0= -github.com/subosito/gotenv v1.2.0 h1:Slr1R9HxAlEKefgq5jn9U+DnETlIUa6HfgEzj0g5d7s= github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= github.com/supranational/blst v0.3.11 h1:LyU6FolezeWAhvQk0k6O/d49jqgO52MSDDfYgbeoEm4= github.com/supranational/blst v0.3.11/go.mod h1:jZJtfjgudtNl4en1tzwPIV3KjUnQUvG3/j+w+fVonLw= @@ -1164,78 +889,54 @@ github.com/swaggo/http-swagger v1.3.0 h1:1+6M4qRorIbdyTWTsGrwnb0r9jGK5dcWN82O6oY github.com/swaggo/http-swagger v1.3.0/go.mod h1:9glekdg40lwclrrKNRGgj/IMDxpNPZ3kzab4oPcF8EM= github.com/swaggo/swag v1.8.3 h1:3pZSSCQ//gAH88lfmxM3Cd1+JCsxV8Md6f36b9hrZ5s= github.com/swaggo/swag v1.8.3/go.mod h1:jMLeXOOmYyjk8PvHTsXBdrubsNd9gUJTTCzL5iBnseg= -github.com/syndtr/goleveldb v1.0.1-0.20200815110645-5c35d600f0ca/go.mod h1:u2MKkTVTVJWe5D1rCvame8WqhBd88EuIwODJZ1VHCPM= -github.com/syndtr/goleveldb v1.0.1-0.20210305035536-64b5b1c73954/go.mod h1:u2MKkTVTVJWe5D1rCvame8WqhBd88EuIwODJZ1VHCPM= -github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7/go.mod h1:q4W45IWZaF22tdD+VEXcAWRA037jwmWEB5VWYORlTpc= github.com/syndtr/goleveldb v1.0.1-0.20220721030215-126854af5e6d h1:vfofYNRScrDdvS342BElfbETmL1Aiz3i2t0zfRj16Hs= github.com/syndtr/goleveldb v1.0.1-0.20220721030215-126854af5e6d/go.mod h1:RRCYJbIwD5jmqPI9XoAFR0OcDxqUctll6zUj/+B4S48= -github.com/templexxx/cpufeat v0.0.0-20180724012125-cef66df7f161/go.mod h1:wM7WEvslTq+iOEAMDLSzhVuOt5BRZ05WirO+b09GHQU= -github.com/templexxx/xor v0.0.0-20191217153810-f85b25db303b/go.mod h1:5XA7W9S6mni3h5uvOC75dA3m9CCCaS83lltmc0ukdi4= github.com/thomaso-mirodin/intmath v0.0.0-20160323211736-5dc6d854e46e h1:cR8/SYRgyQCt5cNCMniB/ZScMkhI9nk8U5C7SbISXjo= github.com/thomaso-mirodin/intmath v0.0.0-20160323211736-5dc6d854e46e/go.mod h1:Tu4lItkATkonrYuvtVjG0/rhy15qrNGNTjPdaphtZ/8= -github.com/tinylib/msgp v1.0.2/go.mod h1:+d+yLhGm8mzTaHzB+wgMYrodPfmZrzkirds8fDWklFE= -github.com/tjfoc/gmsm v1.3.0/go.mod h1:HaUcFuY0auTiaHB9MHFGCPx5IaLhTUd2atbCFBQXn9w= -github.com/tklauser/go-sysconf v0.3.5/go.mod h1:MkWzOF4RMCshBAMXuhXJs64Rte09mITnppBXY/rYEFI= -github.com/tklauser/go-sysconf v0.3.6/go.mod h1:MkWzOF4RMCshBAMXuhXJs64Rte09mITnppBXY/rYEFI= -github.com/tklauser/go-sysconf v0.3.7/go.mod h1:JZIdXh4RmBvZDBZ41ld2bGxRV3n4daiiqA3skYhAoQ4= -github.com/tklauser/go-sysconf v0.3.11 h1:89WgdJhk5SNwJfu+GKyYveZ4IaJ7xAkecBo+KdJV0CM= -github.com/tklauser/go-sysconf v0.3.11/go.mod h1:GqXfhXY3kiPa0nAXPDIQIWzJbMCB7AmcWpGR8lSZfqI= -github.com/tklauser/numcpus v0.2.2/go.mod h1:x3qojaO3uyYt0i56EW/VUYs7uBvdl2fkfZFu0T9wgjM= -github.com/tklauser/numcpus v0.2.3/go.mod h1:vpEPS/JC+oZGGQ/My/vJnNsvMDQL6PwOqt8dsCw5j+E= -github.com/tklauser/numcpus v0.6.0 h1:kebhY2Qt+3U6RNK7UqpYNA+tJ23IBEGKkB7JQBfDYms= -github.com/tklauser/numcpus v0.6.0/go.mod h1:FEZLMke0lhOUG6w2JadTzp0a+Nl8PF/GFkQ5UVIcaL4= +github.com/tklauser/go-sysconf v0.3.12 h1:0QaGUFOdQaIVdPgfITYzaTegZvdCjmYO52cSFAEVmqU= +github.com/tklauser/go-sysconf v0.3.12/go.mod h1:Ho14jnntGE1fpdOqQEEaiKRpvIavV0hSfmBq8nJbHYI= +github.com/tklauser/numcpus v0.6.1 h1:ng9scYS7az0Bk4OZLvrNXNSAO2Pxr1XXRAPyjhIx+Fk= +github.com/tklauser/numcpus v0.6.1/go.mod h1:1XfjsgE2zo8GVw7POkMbHENHzVg3GzmoZ9fESEdAacY= github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/twitchtv/twirp v7.1.0+incompatible/go.mod h1:RRJoFSAmTEh2weEqWtpPE3vFK5YBhA6bqp2l1kfCC5A= -github.com/tyler-smith/go-bip39 v1.0.1-0.20181017060643-dbb3b84ba2ef/go.mod h1:sJ5fKU0s6JVwZjjcUEX2zFOnvq0ASQ2K9Zr6cf67kNs= -github.com/tyler-smith/go-bip39 v1.0.2/go.mod h1:sJ5fKU0s6JVwZjjcUEX2zFOnvq0ASQ2K9Zr6cf67kNs= github.com/tyler-smith/go-bip39 v1.1.0 h1:5eUemwrMargf3BSLRRCalXT93Ns6pQJIjYQN2nyfOP8= -github.com/tyler-smith/go-bip39 v1.1.0/go.mod h1:gUYDtqQw1JS3ZJ8UWVcGTGqqr6YIN3CWg+kkNaLt55U= github.com/uber/jaeger-client-go v2.25.0+incompatible h1:IxcNZ7WRY1Y3G4poYlx24szfsn/3LvK9QHCq9oQw8+U= github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc= github.com/ugorji/go v1.1.7/go.mod h1:kZn38zHttfInRq0xu/PH0az30d+z6vm202qpg1oXVMw= github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= github.com/ugorji/go/codec v1.1.7/go.mod h1:Ax+UKWsSmolVDwsd+7N3ZtXu+yMGCf907BLYF3GoBXY= github.com/umbracle/gohashtree v0.0.2-alpha.0.20230207094856-5b775a815c10 h1:CQh33pStIp/E30b7TxDlXfM0145bn2e8boI30IxAhTg= -github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= github.com/urfave/cli v1.22.12 h1:igJgVw1JdKH+trcLWLeLwZjU9fEfPesQ+9/e4MQ44S8= github.com/urfave/cli v1.22.12/go.mod h1:sSBEIC79qR6OvcmsD4U3KABeOTxDqQtdDnaFuUN30b8= -github.com/urfave/cli/v2 v2.3.0/go.mod h1:LJmUH05zAU44vOAcrfzZQKsZbVcdbOG8rtL3/XcUArI= -github.com/urfave/cli/v2 v2.17.2-0.20221006022127-8f469abc00aa h1:5SqCsI/2Qya2bCzK15ozrqo2sZxkh0FHynJZOTVoV6Q= +github.com/urfave/cli/v2 v2.24.1 h1:/QYYr7g0EhwXEML8jO+8OYt5trPnLHS0p3mrgExJ5NU= github.com/urfave/negroni v1.0.0 h1:kIimOitoypq34K7TG7DUaJ9kq/N4Ofuwi1sjz0KipXc= github.com/urfave/negroni v1.0.0/go.mod h1:Meg73S6kFm/4PpbYdq35yYWoCZ9mS/YSx+lKnmiohz4= github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= +github.com/valyala/fasthttp v1.6.0/go.mod h1:FstJa9V+Pj9vQ7OJie2qMHdwemEDaDiSdBnvPM1Su9w= github.com/valyala/fasttemplate v1.0.1/go.mod h1:UQGH1tvbgY+Nz5t2n7tXsz52dQxojPUpymEIMZ47gx8= github.com/valyala/fasttemplate v1.2.1/go.mod h1:KHLXt3tVN2HBp8eijSv/kGJopbvo7S+qRAEEKiv+SiQ= -github.com/wealdtech/ethereal v0.0.0-20211110093529-ed2beb2b4a38 h1:k5dfyU/CXesxiko9wpZN05blJUIax8mMXGVS7Uk2r/Q= -github.com/wealdtech/ethereal v0.0.0-20211110093529-ed2beb2b4a38/go.mod h1:Gj3SOtZtee/3+3psis6VLuXzGZDWKH0mxorvvJSIjes= -github.com/wealdtech/go-bytesutil v1.1.1/go.mod h1:jENeMqeTEU8FNZyDFRVc7KqBdRKSnJ9CCh26TcuNb9s= +github.com/valyala/tcplisten v0.0.0-20161114210144-ceec8f93295a/go.mod h1:v3UYOV9WzVtRmSR+PDvWpU/qWl4Wa5LApYYX4ZtKbio= github.com/wealdtech/go-bytesutil v1.2.1 h1:TjuRzcG5KaPwaR5JB7L/OgJqMQWvlrblA1n0GfcXFSY= github.com/wealdtech/go-bytesutil v1.2.1/go.mod h1:RhUDUGT1F4UP4ydqbYp2MWJbAel3M+mKd057Pad7oag= -github.com/wealdtech/go-ens/v3 v3.5.1/go.mod h1:bVuYoWYEEeEu7Zy95rIMjPR34QFJarxt8p84ywSo0YM= -github.com/wealdtech/go-ens/v3 v3.5.5 h1:/jq3CDItK0AsFnZtiFJK44JthkAMD5YE3WAJOh4i7lc= -github.com/wealdtech/go-ens/v3 v3.5.5/go.mod h1:w0EDKIm0dIQnqEKls6ORat/or+AVfPEdEXVfN71EeEE= -github.com/wealdtech/go-erc1820 v1.2.3 h1:BmA93UGhZhAKN1AJoZ74JyQ+N93CBgNIsD8MZUjGbWY= -github.com/wealdtech/go-erc1820 v1.2.3/go.mod h1:iAQx1hyjt3165fwe/2oTUmysKQYZSJ46HkMPsaRxVew= -github.com/wealdtech/go-eth2-types/v2 v2.5.3/go.mod h1:JCIc6C8SAaxQPasY2q3g7iFUuebmKURb+TyqCKrmq4c= -github.com/wealdtech/go-eth2-types/v2 v2.6.0/go.mod h1:psOez/ZRBzZSDl5hiNDwRf5ZqQujNE6h5FxAz09Koxg= +github.com/wealdtech/go-ens/v3 v3.6.0 h1:EAByZlHRQ3vxqzzwNi0GvEq1AjVozfWO4DMldHcoVg8= +github.com/wealdtech/go-ens/v3 v3.6.0/go.mod h1:hcmMr9qPoEgVSEXU2Bwzrn/9NczTWZ1rE53jIlqUpzw= github.com/wealdtech/go-eth2-types/v2 v2.8.1 h1:y2N3xSIZ3tVqsnvj4AgPkh48U5sM612vhZwlK3k+3lM= github.com/wealdtech/go-eth2-types/v2 v2.8.1/go.mod h1:3TJShI4oBzG8pCZsfe3NZAq8QAmXrC2rd45q7Vn/XB8= -github.com/wealdtech/go-eth2-util v1.6.4/go.mod h1:OB2m2u3kO3yVqJA3v1jVaWH1Stv5cBmAlBFX47cokRI= github.com/wealdtech/go-eth2-util v1.8.1 h1:nb50hygsNoql94akg7GN6im/weg8ZZgJWHgiyrj8qiU= github.com/wealdtech/go-eth2-util v1.8.1/go.mod h1:vv+8jVgYRXEGty/VLPNn1RYlbQNYmTht3VR6nfh0z4E= github.com/wealdtech/go-multicodec v1.4.0 h1:iq5PgxwssxnXGGPTIK1srvt6U5bJwIp7k6kBrudIWxg= github.com/wealdtech/go-multicodec v1.4.0/go.mod h1:aedGMaTeYkIqi/KCPre1ho5rTb3hGpu/snBOS3GQLw4= -github.com/wealdtech/go-string2eth v1.1.0 h1:USJQmysUrBYYmZs7d45pMb90hRSyEwizP7lZaOZLDAw= -github.com/wealdtech/go-string2eth v1.1.0/go.mod h1:RUzsLjJtbZaJ/3UKn9kY19a/vCCUHtEWoUW3uiK6yGU= -github.com/willf/bitset v1.1.3/go.mod h1:RjeCKbqT1RxIR/KWY6phxZiaY1IyutSBfGjNPySAYV4= -github.com/wsddn/go-ecdh v0.0.0-20161211032359-48726bab9208/go.mod h1:IotVbo4F+mw0EzQ08zFqg7pK3FebNXpaMsRy2RT+Ees= -github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c/go.mod h1:lB8K/P019DLNhemzwFU4jHLhdvlE6uDZjXFejJXr49I= -github.com/xdg/stringprep v1.0.0/go.mod h1:Jhud4/sHMO4oL310DaZAKk9ZaJ08SJfe+sJh0HrGL1Y= +github.com/wealdtech/go-string2eth v1.2.1 h1:u9sofvGFkp+uvTg4Nvsvy5xBaiw8AibGLLngfC4F76g= +github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= +github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ= +github.com/xeipuuv/gojsonschema v1.2.0/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQluxsYJ78Id3Y= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= -github.com/xlab/treeprint v0.0.0-20180616005107-d6fb6747feb6/go.mod h1:ce1O1j6UtZfjr22oyGxGLbauSBp2YVXpARAosm7dHBg= +github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 h1:bAn7/zixMGCfxrRTfdpNzjtPYqr8smhKouy9mxVdGPU= -github.com/xtaci/kcp-go v5.4.20+incompatible/go.mod h1:bN6vIwHQbfHaHtFpEssmWsN45a+AZwO7eyRCmEIbtvE= -github.com/xtaci/lossyconn v0.0.0-20190602105132-8df528c0c9ae/go.mod h1:gXtu8J62kEgmN++bm9BVICuT/e8yiLI2KFobd/TRFsE= +github.com/yalp/jsonpath v0.0.0-20180802001716-5cc68e5049a0/go.mod h1:/LWChgwKmvncFJFHJ7Gvn9wZArjbV5/FppcK2fKk/tI= +github.com/yudai/gojsondiff v1.0.0/go.mod h1:AY32+k2cwILAkW1fbgxQ5mUmMiZFgLIV+FBNExI05xg= +github.com/yudai/golcs v0.0.0-20170316035057-ecda9a501e82/go.mod h1:lgjkn3NuSvDfVJdfcVVdX+jpBxNmX4rDAzaS45IcYoM= +github.com/yudai/pp v2.0.1+incompatible/go.mod h1:PuxR/8QJ7cyCkFp/aUDS+JY727OFEZkTdatxwunjIkc= github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= @@ -1243,15 +944,12 @@ github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9dec github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/yuin/goldmark v1.4.0/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= -github.com/yusufpapurcu/wmi v1.2.3 h1:E1ctvB7uKFMOJw3fdOW32DwGE9I7t++CRUEMKvFoFiw= -github.com/yusufpapurcu/wmi v1.2.3/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= +github.com/yusufpapurcu/wmi v1.2.2 h1:KBNDSne4vP5mbSWnJbO+51IMOXJB67QiYCSBrubbPRg= +github.com/yusufpapurcu/wmi v1.2.2/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= github.com/zenazn/goji v0.9.0/go.mod h1:7S9M489iMyHBNxwZnk9/EHS098H4/F6TATF2mIxtB1Q= github.com/zesik/proxyaddr v0.0.0-20161218060608-ec32c535184d h1:Gsw/uTjNB2vkIEhBO3NAXjKo6QRY6D5B0GzMv980ses= github.com/zesik/proxyaddr v0.0.0-20161218060608-ec32c535184d/go.mod h1:PXKs8dwJ+2BKBQxMlKg+eaO27VucEN/jrrymnRB5tvs= go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= -go.etcd.io/etcd/api/v3 v3.5.0/go.mod h1:cbVKeC6lCfl7j/8jBhAK6aIYO9XOjdptoxU/nLQcPvs= -go.etcd.io/etcd/client/pkg/v3 v3.5.0/go.mod h1:IJHfcCEKxYu1Os13ZdwCwIUTUVGYTSAM3YSwc9/Ac1g= -go.etcd.io/etcd/client/v2 v2.305.0/go.mod h1:h9puh54ZTgAKtEbut2oe9P4L/oqKCVB6xsXlzd7alYQ= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= @@ -1259,7 +957,6 @@ go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= go.opencensus.io v0.22.6/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E= -go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E= go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= @@ -1277,8 +974,6 @@ go.uber.org/zap v1.9.1/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= go.uber.org/zap v1.13.0/go.mod h1:zwrFLgMcdUuIBviXEYEH1YKNaOBnKXsx2IPda5bBwHM= go.uber.org/zap v1.16.0/go.mod h1:MA8QOfq0BHJwdXa996Y4dYkAqRKB8/1K1QMMZVaNZjQ= -go.uber.org/zap v1.17.0/go.mod h1:MXVU+bhUf/A7Xi2HNOnopQOrmycQ5Ih87HtOu4q5SSo= -golang.org/x/crypto v0.0.0-20170930174604-9419663f5a44/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= @@ -1286,37 +981,23 @@ golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACk golang.org/x/crypto v0.0.0-20190411191339-88737f569e3a/go.mod h1:WFFai1msRO1wXaEeE5yQxYXgSfI8pQAWXbQop6sCtWE= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20190909091759-094676da4a83/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20191219195013-becbf705a915/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20200115085410-6d4e4cb37c7d/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20200204104054-c9f3fb736b72/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20200221231518-2aa609cf4a9d/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20200510223506-06a226fb4e37/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20191227163750-53104e6ec876/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20200820211705-5c72a883971a/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20201016220609-9e8e0b390897/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20201203163018-be400aefbc4c/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= -golang.org/x/crypto v0.0.0-20201221181555-eec23a3978ad/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= -golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= golang.org/x/crypto v0.0.0-20210616213533-5ff15b29337e/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.0.0-20211108221036-ceb1ce70b4fa/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20211215165025-cf75a172585e/go.mod h1:P+XmwS30IXTQdn5tA2iutPOUgjI07+tq3H3K9MVA1s8= golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58= -golang.org/x/crypto v0.7.0 h1:AvwMYaRytfdeVt3u6mLaxYtErKYjxA2OXjJ1HHq6t3A= -golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU= -golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= -golang.org/x/exp v0.0.0-20180807140117-3d87b88a115f/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/crypto v0.13.0 h1:mvySKfSWJ+UKUii46M40LOvyWfN0s2U+46/jDd0e6Ck= +golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= -golang.org/x/exp v0.0.0-20190125153040-c74c464bbbf2/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= -golang.org/x/exp v0.0.0-20190731235908-ec7cb31e5a56/go.mod h1:JhuoJpWY28nO4Vef9tZUw9qufEGTyX1+7lmHxV5q5G4= golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek= golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY= golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= @@ -1324,9 +1005,8 @@ golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u0 golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= -golang.org/x/exp v0.0.0-20230206171751-46f607a40771 h1:xP7rWLUr1e1n2xkK5YB4LI0hPEy3LJC6Wk+D4pGlOJg= -golang.org/x/exp v0.0.0-20230206171751-46f607a40771/go.mod h1:CxIveKay+FTh1D0yPZemJVgC/95VzuuOLq5Qi4xnoYc= -golang.org/x/image v0.0.0-20180708004352-c73c2afc3b81/go.mod h1:ux5Hcp/YLpHSI86hEcLt0YII63i6oz57MZXIpbrjZUs= +golang.org/x/exp v0.0.0-20230905200255-921286631fa9 h1:GoHiUyI/Tp2nVkLI2mCxVkOjsbSXD66ic0XW0js0R9g= +golang.org/x/exp v0.0.0-20230905200255-921286631fa9/go.mod h1:S2oDrQGGwySpoQPVqRShND87VCbxmc6bL1Yd2oYrm6k= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= @@ -1343,25 +1023,19 @@ golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5/go.mod h1:3xt1FjdF8hUf6vQPI golang.org/x/lint v0.0.0-20210508222113-6edffad5e616/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= -golang.org/x/mobile v0.0.0-20200801112145-973feb4309de/go.mod h1:skQtrUTUwhdJvXM/2KKJzY8pDgNr9I/FOMqDVRPBUS4= golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= -golang.org/x/mod v0.1.1-0.20191209134235-331c550502dd/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= -golang.org/x/mod v0.9.0 h1:KENHtAZL2y3NLMYZeHY9DW8HW8V+kQyJsY/V9JlKvCs= -golang.org/x/mod v0.9.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= -golang.org/x/net v0.0.0-20180719180050-a680a1efc54d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/mod v0.12.0 h1:rmsUpXtvNzj340zd98LZ4KntptpfRHwpFOHG188oHXc= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20181011144130-49bb7cea24b1/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -1395,23 +1069,18 @@ golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/ golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= -golang.org/x/net v0.0.0-20200813134508-3edf25e44fcc/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20200927032502-5d4f70055728/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.0.0-20201026091529-146b70c837a4/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201031054903-ff519b6c9102/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201209123823-ac852fbbde11/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= -golang.org/x/net v0.0.0-20210220033124-5f55cee0dc0d/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4/go.mod h1:RBQZq4jEuRlivfhVLdyRGr576XBO4/greRjx4P4O3yc= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk= golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20210614182718-04defd469f4e/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20210726213435-c6fcb2dbf985/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211008194852-3b03d305991f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= @@ -1429,10 +1098,6 @@ golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4Iltr golang.org/x/oauth2 v0.0.0-20200902213428-5d25da1a8d43/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20201109201403-9fd604954f58/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20201208152858-08078c50e5b5/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20210218202405-ba52d332ba99/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20210220000619-9bb904979d93/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20210313182246-cd4f82c27b84/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20210402161424-2e8d93401602/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20210427180440-81ed05c6b58c/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.3.0 h1:6l90koy8/LaBLmLu8jpHeHexzMwEita0zFfYlggy2F8= @@ -1455,7 +1120,6 @@ golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20180926160741-c2ed4eda69e7/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181026203630-95b1ffbd15a5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -1483,13 +1147,11 @@ golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200107162124-548cf772de50/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200219091948-cb0a6d8edb6c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -1498,45 +1160,32 @@ golang.org/x/sys v0.0.0-20200501052902-10377860bb8e/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200501145240-bc7a7d42d5c3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200814200057-3d37ad5750ed/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200824131525-c12d262b63d8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200826173525-f9321e4c35a6/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20201026173827-119d4633e4d1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201101102859-da207088b7d1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201214095126-aec9a390925b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210104204734-6f8348627aad/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210220050731-9a76102bfb43/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210303074136-134d130e1a04/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210305230114-8fe3ee5dd75b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210309074719-68d13333faf2/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210315160823-c6e025ad8005/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210316164454-77fc1eacc6aa/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210324051608-47abb6519492/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210420205809-ac73e9fd8988/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210816183151-1e6c022a8912/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20211109184856-51b60fd695b3/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220209214540-3681064d5158/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -1577,14 +1226,11 @@ golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxb golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20201208040808-7e3f01d25324/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20220922220347-f3bd1da661af h1:Yx9k8YCG3dvF87UAn2tu2HQLf2dt/eR1bXxpLMWeH+Y= +golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4= golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20180525024113-a5b4c53f6e8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20181221001348-537d06c36207/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20190206041539-40960b6deb8e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= @@ -1613,15 +1259,12 @@ golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtn golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200103221440-774c71fcf114/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200108203644-89082a384178/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200117012304-6edc0a871e69/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200117161641-43d50277825c/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200122220014-bf1340f18c4a/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200204074204-1cc6d1ef6c74/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200221224223-e1da425f72fd/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200227222343-706bc42d1f0d/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= @@ -1643,11 +1286,9 @@ golang.org/x/tools v0.0.0-20201201161351-ac6f37ff4c2a/go.mod h1:emZCQorbCU4vsT4f golang.org/x/tools v0.0.0-20201202200335-bef1c476418a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20201208233053-a543418bbed2/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.0.0-20210105154028-b0ab187a4818/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= -golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= -golang.org/x/tools v0.1.6-0.20210726203631-07bc1bf47fb2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.1.3/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.7/go.mod h1:LGqMHiF4EqQNHR1JncWGqT5BVaXmza+X+BDGol+dOxo= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/tools v0.13.0 h1:Iey4qkscZuv0VvIt8E0neZjtPVQFSc870HQ448QgEmQ= @@ -1661,14 +1302,8 @@ golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20220517211312-f3a8303e98df/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8= golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 h1:H2TDz8ibqkAF6YGhCdN3jS9O0/s90v0rJh3X/OLHEUk= golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8= -gonum.org/v1/gonum v0.0.0-20180816165407-929014505bf4/go.mod h1:Y+Yx5eoAFn32cQvJDxZx5Dpnq+c3wtXuadVZAcxbbBo= -gonum.org/v1/gonum v0.0.0-20181121035319-3f7ecaa7e8ca/go.mod h1:Y+Yx5eoAFn32cQvJDxZx5Dpnq+c3wtXuadVZAcxbbBo= -gonum.org/v1/gonum v0.6.0/go.mod h1:9mxDZsDKxgMAuccQkewq682L+0eCu4dCN2yonUJTCLU= gonum.org/v1/gonum v0.12.0 h1:xKuo6hzt+gMav00meVPUlXwSdoEJP46BR+wdxQEFK2o= gonum.org/v1/gonum v0.12.0/go.mod h1:73TDxJfAAHeA8Mk9mf8NlIppyhQNo5GLTcYeqgo2lvY= -gonum.org/v1/netlib v0.0.0-20181029234149-ec6d1f5cefe6/go.mod h1:wa6Ws7BG/ESfp6dHfk7C6KdzKA7wR7u/rKwOGE66zvw= -gonum.org/v1/netlib v0.0.0-20190313105609-8cb42192e0e0/go.mod h1:wa6Ws7BG/ESfp6dHfk7C6KdzKA7wR7u/rKwOGE66zvw= -gonum.org/v1/plot v0.0.0-20190515093506-e2840ee46a6b/go.mod h1:Wt8AAjI+ypCyYX3nZBvf6cAIx93T+c/OS2HFAYskSZc= google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= @@ -1687,10 +1322,6 @@ google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSr google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc= google.golang.org/api v0.35.0/go.mod h1:/XrVsuzM0rZmrsbjJutiuftIzeuTQcEeaYcSk/mQ1dg= google.golang.org/api v0.36.0/go.mod h1:+z5ficQTmoYpPn8LCUNVpK5I7hwkpjbcgqA7I34qYtE= -google.golang.org/api v0.40.0/go.mod h1:fYKFpnQN0DsDSKRVRcQSDQNtqWPfM9i+zNPxepjRCQ8= -google.golang.org/api v0.41.0/go.mod h1:RkxM5lITDfTzmyKFPt+wGrCJbVfniCr2ool8kTBzRTU= -google.golang.org/api v0.43.0/go.mod h1:nQsDGjRXMo4lvh5hP0TKqF244gqhGcr/YSIykhUk/94= -google.golang.org/api v0.44.0/go.mod h1:EBOGZqzyhtvMDoxwS97ctnh0zUmYY6CxqXsc1AvkYD8= google.golang.org/api v0.102.0 h1:JxJl2qQ85fRMPNvlZY/enexbxpCjLwGhZUtgfGeQ51I= google.golang.org/api v0.102.0/go.mod h1:3VFl6/fzoA+qNuS1N1/VfXY4LjoXN/wzeIp7TweWwGo= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= @@ -1707,7 +1338,6 @@ google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRn google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190716160619-c506a9f90610/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= @@ -1715,12 +1345,10 @@ google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvx google.golang.org/genproto v0.0.0-20191115194625-c23dd37a84c9/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= google.golang.org/genproto v0.0.0-20191216164720-4f79533eabd1/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20200108215221-bd8f9a0ef82f/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= google.golang.org/genproto v0.0.0-20200115191322-ca5a22157cba/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= google.golang.org/genproto v0.0.0-20200122232147-0452cf42e150/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= google.golang.org/genproto v0.0.0-20200204135345-fa8e72b47b90/go.mod h1:GmwEX6Z4W5gMy59cAlVYjN9JhxgbQH6Gn+gFDQe2lzA= google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200218151345-dad8c97a84f5/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200224152610-e50cd9704f63/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200228133532-8c2c7df3a383/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200305110556-506484158171/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= @@ -1741,15 +1369,9 @@ google.golang.org/genproto v0.0.0-20201201144952-b05cb90ed32e/go.mod h1:FWY/as6D google.golang.org/genproto v0.0.0-20201203001206-6486ece9c497/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20201210142538-e3217bee35cc/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20201211151036-40ec1c210f7a/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20201214200347-8c77b98c765d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210207032614-bba0dbe2a9ea/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20210222152913-aa3ee6e6a81c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20210303154014-9728d6b83eeb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20210310155132-4ce2db91004e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20210319143718-93e7006c17a6/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20210402141018-6c239bbf2bb1/go.mod h1:9lPAdzaEmUacj36I+k7YKbEc5CXzPIeORRgDAUOu28A= google.golang.org/genproto v0.0.0-20210426193834-eac7f76ac494/go.mod h1:P3QM42oQyzQSnHPnZ/vqoCdDmzH28fzWByN9asMeM8A= -google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= +google.golang.org/genproto v0.0.0-20210624195500-8bfb893ecb84/go.mod h1:SzzZ/N+nwJDaO1kznhnlzqS8ocJICar6hYhVyhi++24= google.golang.org/genproto v0.0.0-20221118155620-16455021b5e6 h1:a2S6M0+660BgMNl++4JPlcAO/CjkqYItDEZwkoDQK7c= google.golang.org/genproto v0.0.0-20221118155620-16455021b5e6/go.mod h1:rZS5c/ZVYMaOGBfO68GWtjOw/eLaZM1X6iVtgjZ+EWg= google.golang.org/grpc v1.12.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= @@ -1770,7 +1392,6 @@ google.golang.org/grpc v1.33.1/go.mod h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTp google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= google.golang.org/grpc v1.34.0/go.mod h1:WotjhfgOW/POjDeRt8vscBtXq+2VjORFy659qA51WJ8= google.golang.org/grpc v1.35.0-dev.0.20201218190559-666aea1fb34c/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= -google.golang.org/grpc v1.35.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= google.golang.org/grpc v1.36.1/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= google.golang.org/grpc v1.37.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= @@ -1796,7 +1417,6 @@ google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQ google.golang.org/protobuf v1.28.1 h1:d0NfwRgPtno5B1Wa6L2DAG+KivqkdutMf1UhdNx175w= google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= -gopkg.in/bsm/ratelimit.v1 v1.0.0-20160220154919-db14e161995a/go.mod h1:KF9sEfUPAXdG8Oev9e99iLGnl2uJMjc5B+4y3O7x610= gopkg.in/cenkalti/backoff.v1 v1.1.0 h1:Arh75ttbsvlpVA7WtVpH4u9h6Zl46xuptxqLxPiSo4Y= gopkg.in/cenkalti/backoff.v1 v1.1.0/go.mod h1:J6Vskwqd+OMVJl8C33mmtxTBs2gyzfv7UDAkHu8BrjI= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= @@ -1804,7 +1424,6 @@ gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8 gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= -gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/go-playground/assert.v1 v1.2.1/go.mod h1:9RXL0bg/zibRAgZUYszZSwO/z8Y/a8bDuhia5mkpMnE= @@ -1815,29 +1434,21 @@ gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/ini.v1 v1.51.1/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/ini.v1 v1.61.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= -gopkg.in/ini.v1 v1.62.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= -gopkg.in/jcmturner/aescts.v1 v1.0.1/go.mod h1:nsR8qBOg+OucoIW+WMhB3GspUQXq9XorLnQb9XtvcOo= -gopkg.in/jcmturner/dnsutils.v1 v1.0.1/go.mod h1:m3v+5svpVOhtFAP/wSz+yzh4Mc0Fg7eRhxkJMWSIz9Q= -gopkg.in/jcmturner/goidentity.v3 v3.0.0/go.mod h1:oG2kH0IvSYNIu80dVAyu/yoefjq1mNfM5bm88whjWx4= -gopkg.in/jcmturner/gokrb5.v7 v7.5.0/go.mod h1:l8VISx+WGYp+Fp7KRbsiUuXTTOnxIc3Tuvyavf11/WM= -gopkg.in/jcmturner/rpc.v1 v1.1.0/go.mod h1:YIdkC4XfD6GXbzje11McwsDuOlZQSb9W4vfLvuNnlv8= +gopkg.in/mgo.v2 v2.0.0-20180705113604-9856a29383ce/go.mod h1:yeKp02qBN3iKW1OzL3MGk2IdtZzaj7SFntXj72NppTA= +gopkg.in/natefinch/lumberjack.v2 v2.0.0 h1:1Lc07Kr7qY4U2YPouBjpCLxpiyxIVoxqXgkXLknAOE8= gopkg.in/natefinch/npipe.v2 v2.0.0-20160621034901-c1b8fa8bdcce h1:+JknDZhAj8YMt7GC73Ei8pv4MzjDUNPHgQWJdtMAaDU= gopkg.in/natefinch/npipe.v2 v2.0.0-20160621034901-c1b8fa8bdcce/go.mod h1:5AcXVHNjg+BDxry382+8OKon8SEWiKktQR07RKPsv1c= -gopkg.in/olebedev/go-duktape.v3 v3.0.0-20200619000410-60c24ae608a6/go.mod h1:uAJfkITjFhyEEuUfm7bsmCZRbW5WRq8s9EY8HZ6hCns= -gopkg.in/redis.v4 v4.2.4/go.mod h1:8KREHdypkCEojGKQcjMqAODMICIVwZAONWq8RowTITA= gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= -gopkg.in/urfave/cli.v1 v1.20.0/go.mod h1:vuBzUtMdQeixQj8LVd+/98pzhxNGQoyuPBlsXHOQNO0= gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= @@ -1857,16 +1468,14 @@ honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWh honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= -honnef.co/go/tools v0.1.3/go.mod h1:NgwopIslSNH47DimFoV78dnkksY2EFtX0ajyb3K/las= k8s.io/apimachinery v0.25.0 h1:MlP0r6+3XbkUG2itd6vp3oxbtdQLQI94fD5gCS+gnoU= k8s.io/client-go v0.25.0 h1:CVWIaCETLMBNiTUta3d5nzRbXvY5Hy9Dpl+VvREpu5E= k8s.io/client-go v0.25.0/go.mod h1:lxykvypVfKilxhTklov0wz1FoaUZ8X4EwbhS6rpRfN8= k8s.io/klog v1.0.0 h1:Pt+yjF5aB1xDSVbau4VsWe+dQNzA0qv1LlXdC2dF6Q8= k8s.io/klog/v2 v2.70.1 h1:7aaoSdahviPmR+XkS7FyxlkkXs6tHISSG03RxleQAVQ= k8s.io/utils v0.0.0-20220728103510-ee6ede2d64ed h1:jAne/RjBTyawwAy0utX5eqigAwz/lQhTmy+Hr/Cpue4= -launchpad.net/gocheck v0.0.0-20140225173054-000000000087/go.mod h1:hj7XX3B/0A+80Vse0e+BUHsHMTEhd0O4cpUHr/e/BUM= -lukechampine.com/blake3 v1.1.7 h1:GgRMhmdsuK8+ii6UZFDL8Nb+VyMwadAgcJyfYHxG6n0= -lukechampine.com/blake3 v1.1.7/go.mod h1:tkKEOtDkNtklkXtLNEOGNq5tcV90tJiA1vAA12R78LA= +lukechampine.com/blake3 v1.2.1 h1:YuqqRuaqsGV71BV/nm9xlI0MKUv4QC54jQnBChWbGnI= +lukechampine.com/blake3 v1.2.1/go.mod h1:0OFRp7fBtAylGVCO40o87sbupkyIGgbpv1+M1k1LM6k= lukechampine.com/uint128 v1.2.0 h1:mBi/5l91vocEN8otkC5bDLhi2KdCticRiwbdB0O+rjI= modernc.org/cc/v3 v3.40.0 h1:P3g79IUS/93SYhtoeaHW+kRCIrYaxJ27MFPv+7kaTOw= modernc.org/ccgo/v3 v3.16.13 h1:Mkgdzl46i5F/CNR/Kj80Ri59hC8TKAhZrYSaqvkwzUw= @@ -1879,7 +1488,6 @@ modernc.org/strutil v1.1.3 h1:fNMm+oJklMGYfU9Ylcywl0CO5O6nTfaowNsh2wpPjzY= modernc.org/token v1.1.0 h1:Xl7Ap9dKaEs5kLoOQeQmPWevfnk/DM5qcLcYlA8ys6Y= rsc.io/binaryregexp v0.2.0 h1:HfqmD5MEmC0zvwBuF187nq9mdnXjXsSivRiXN7SmRkE= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= -rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= rsc.io/tmplfunc v0.0.3 h1:53XFQh69AfOa8Tw0Jm7t+GV7KZhOi6jzsCzTtKbMvzU= From 4f5f228cd70763d3c12109d2cf749796d810235b Mon Sep 17 00:00:00 2001 From: peter <1674920+peterbitfly@users.noreply.github.com> Date: Tue, 12 Sep 2023 11:09:44 +0200 Subject: [PATCH 104/106] (BIDS-2379) adjust the effective & balance distribution chart to source data from the node instead of bigtable --- cmd/frontend-data-updater/main.go | 9 ++++++++ rpc/client.go | 3 +++ rpc/lighthouse.go | 20 ++++++++++++++++++ services/charts_updater.go | 34 ++++++++++++------------------- 4 files changed, 45 insertions(+), 21 deletions(-) create mode 100644 rpc/client.go diff --git a/cmd/frontend-data-updater/main.go b/cmd/frontend-data-updater/main.go index 5955c3392f..3c81423f8b 100644 --- a/cmd/frontend-data-updater/main.go +++ b/cmd/frontend-data-updater/main.go @@ -3,12 +3,14 @@ package main import ( "eth2-exporter/cache" "eth2-exporter/db" + "eth2-exporter/rpc" "eth2-exporter/services" "eth2-exporter/types" "eth2-exporter/utils" "eth2-exporter/version" "flag" "fmt" + "math/big" _ "github.com/jackc/pgx/v4/stdlib" "github.com/sirupsen/logrus" @@ -76,6 +78,13 @@ func main() { logrus.Fatalf("No cache provider set. Please set TierdCacheProvider (example redis, bigtable)") } + chainID := new(big.Int).SetUint64(utils.Config.Chain.Config.DepositChainID) + rpcClient, err := rpc.NewLighthouseClient("http://"+cfg.Indexer.Node.Host+":"+cfg.Indexer.Node.Port, chainID) + if err != nil { + utils.LogFatal(err, "new explorer lighthouse client error", 0) + } + rpc.CurrentClient = rpcClient + logrus.Infof("initializing frontend services") services.Init() // Init frontend services logrus.Infof("frontend services initiated") diff --git a/rpc/client.go b/rpc/client.go new file mode 100644 index 0000000000..adb62de71e --- /dev/null +++ b/rpc/client.go @@ -0,0 +1,3 @@ +package rpc + +var CurrentClient *LighthouseClient diff --git a/rpc/lighthouse.go b/rpc/lighthouse.go index 037be9a927..52f938d492 100644 --- a/rpc/lighthouse.go +++ b/rpc/lighthouse.go @@ -258,6 +258,26 @@ func (lc *LighthouseClient) GetEpochAssignments(epoch uint64) (*types.EpochAssig return assignments, nil } +func (lc *LighthouseClient) GetValidatorState(epoch uint64) (*StandardValidatorsResponse, error) { + validatorsResp, err := lc.get(fmt.Sprintf("%s/eth/v1/beacon/states/%d/validators", lc.endpoint, epoch*utils.Config.Chain.Config.SlotsPerEpoch)) + if err != nil && epoch == 0 { + validatorsResp, err = lc.get(fmt.Sprintf("%s/eth/v1/beacon/states/%v/validators", lc.endpoint, "genesis")) + if err != nil { + return nil, fmt.Errorf("error retrieving validators for genesis: %v", err) + } + } else if err != nil { + return nil, fmt.Errorf("error retrieving validators for epoch %v: %v", epoch, err) + } + + parsedValidators := &StandardValidatorsResponse{} + err = json.Unmarshal(validatorsResp, parsedValidators) + if err != nil { + return nil, fmt.Errorf("error parsing epoch validators: %v", err) + } + + return parsedValidators, nil +} + // GetEpochData will get the epoch data from Lighthouse RPC api func (lc *LighthouseClient) GetEpochData(epoch uint64, skipHistoricBalances bool) (*types.EpochData, error) { wg := &errgroup.Group{} diff --git a/services/charts_updater.go b/services/charts_updater.go index b620bb6bed..60f2d89ce2 100644 --- a/services/charts_updater.go +++ b/services/charts_updater.go @@ -4,6 +4,7 @@ import ( "eth2-exporter/cache" "eth2-exporter/db" "eth2-exporter/metrics" + "eth2-exporter/rpc" "eth2-exporter/types" "eth2-exporter/utils" "fmt" @@ -599,23 +600,18 @@ func balanceDistributionChartData() (*types.GenericChartData, error) { return nil, fmt.Errorf("chart-data not available pre-genesis") } - validators, err := db.GetValidatorIndices() + validators, err := rpc.CurrentClient.GetValidatorState(epoch) if err != nil { return nil, err } - balances, err := db.BigtableClient.GetValidatorBalanceHistory(validators, epoch, epoch) - if err != nil { - return nil, err + if validators.Data == nil { + return nil, fmt.Errorf("GetValidatorState returned empty validator set for epoch %v", epoch) } - currentBalances := make([]float64, 0, len(balances)) - - for _, balance := range balances { - if len(balance) == 0 { - continue - } - currentBalances = append(currentBalances, float64(balance[0].Balance)/1e9) + currentBalances := make([]float64, 0, len(validators.Data)) + for _, entry := range validators.Data { + currentBalances = append(currentBalances, float64(entry.Balance)/1e9) } bins := int(math.Sqrt(float64(len(currentBalances)))) + 1 @@ -654,23 +650,19 @@ func effectiveBalanceDistributionChartData() (*types.GenericChartData, error) { return nil, fmt.Errorf("chart-data not available pre-genesis") } - validators, err := db.GetValidatorIndices() + validators, err := rpc.CurrentClient.GetValidatorState(epoch) if err != nil { return nil, err } - balances, err := db.BigtableClient.GetValidatorBalanceHistory(validators, epoch, epoch) - if err != nil { - return nil, err + if validators.Data == nil { + return nil, fmt.Errorf("GetValidatorState returned empty validator set for epoch %v", epoch) } - effectiveBalances := make([]float64, 0, len(balances)) + effectiveBalances := make([]float64, 0, len(validators.Data)) - for _, balance := range balances { - if len(balance) == 0 { - continue - } - effectiveBalances = append(effectiveBalances, float64(balance[0].EffectiveBalance)/1e9) + for _, entry := range validators.Data { + effectiveBalances = append(effectiveBalances, float64(entry.Validator.EffectiveBalance)/1e9) } bins := int(math.Sqrt(float64(len(effectiveBalances)))) + 1 From e21356055d2f058d3e613ba3236d60e38c5c96ce Mon Sep 17 00:00:00 2001 From: peter <1674920+peterbitfly@users.noreply.github.com> Date: Tue, 12 Sep 2023 12:13:29 +0200 Subject: [PATCH 105/106] (BIDS-2379) improve fetching the last block of the data table --- db/bigtable_eth1.go | 4 +--- services/services.go | 6 ++---- 2 files changed, 3 insertions(+), 7 deletions(-) diff --git a/db/bigtable_eth1.go b/db/bigtable_eth1.go index 39fc68933d..849af75237 100644 --- a/db/bigtable_eth1.go +++ b/db/bigtable_eth1.go @@ -400,8 +400,6 @@ func (bigtable *Bigtable) GetMostRecentBlockFromDataTable() (*types.Eth1BlockInd prefix := fmt.Sprintf("%s:B:", bigtable.chainId) rowRange := gcp_bigtable.PrefixRange(prefix) - rowFilter := gcp_bigtable.RowFilter(gcp_bigtable.ColumnFilter("d")) - block := types.Eth1BlockIndexed{} rowHandler := func(row gcp_bigtable.Row) bool { @@ -421,7 +419,7 @@ func (bigtable *Bigtable) GetMostRecentBlockFromDataTable() (*types.Eth1BlockInd return c == 0 } - err := bigtable.tableData.ReadRows(ctx, rowRange, rowHandler, rowFilter) + err := bigtable.tableData.ReadRows(ctx, rowRange, rowHandler, gcp_bigtable.LimitRows(2), gcp_bigtable.RowFilter(gcp_bigtable.ColumnFilter("d"))) if err != nil { return nil, err } diff --git a/services/services.go b/services/services.go index e5460fb241..998f5444ad 100644 --- a/services/services.go +++ b/services/services.go @@ -1482,24 +1482,22 @@ func mempoolUpdater(wg *sync.WaitGroup) { func burnUpdater(wg *sync.WaitGroup) { firstRun := true - for { + for ; ; time.Sleep(time.Minute * 15) { // only update once every 15 minutes data, err := getBurnPageData() if err != nil { logger.Errorf("error retrieving burn page data: %v", err) - time.Sleep(time.Second * 30) continue } cacheKey := fmt.Sprintf("%d:frontend:burn", utils.Config.Chain.Config.DepositChainID) err = cache.TieredCache.Set(cacheKey, data, time.Hour*24) if err != nil { - logger.Errorf("error caching relaysData: %v", err) + logger.Errorf("error caching burn data: %v", err) } if firstRun { logger.Infof("initialized burn updater") wg.Done() firstRun = false } - time.Sleep(time.Minute * 15) // only update once every 15 minutes } } From a4a2c7029f370dc8a2d914d3a5d10c60bb59c37f Mon Sep 17 00:00:00 2001 From: peter <1674920+peterbitfly@users.noreply.github.com> Date: Tue, 12 Sep 2023 13:05:28 +0200 Subject: [PATCH 106/106] (BIDS-2379) split max block number for el and cl blocks --- db/bigtable.go | 37 +++++++++++++++++++------------------ db/bigtable_eth1.go | 12 ++++++------ 2 files changed, 25 insertions(+), 24 deletions(-) diff --git a/db/bigtable.go b/db/bigtable.go index 592972f236..a056d5e869 100644 --- a/db/bigtable.go +++ b/db/bigtable.go @@ -39,8 +39,9 @@ const ( SUM_COLUMN = "sum" - MAX_BLOCK_NUMBER = 1000000000 - 1 - MAX_EPOCH = 1000000000 - 1 + MAX_CL_BLOCK_NUMBER = 1000000000 - 1 + MAX_EL_BLOCK_NUMBER = 1000000000 + MAX_EPOCH = 1000000000 - 1 MAX_BATCH_MUTATIONS = 100000 @@ -633,11 +634,11 @@ func (bigtable *Bigtable) SaveAttestationDuties(duties map[types.Slot]map[types. epoch := utils.EpochOfSlot(uint64(attestedSlot)) bigtable.lastAttestationCacheMux.Lock() if len(inclusions) == 0 { // for missed attestations we write the max block number which will yield a cell ts of 0 - inclusions = append(inclusions, MAX_BLOCK_NUMBER) + inclusions = append(inclusions, MAX_CL_BLOCK_NUMBER) } for _, inclusionSlot := range inclusions { mutInclusionSlot := gcp_bigtable.NewMutation() - mutInclusionSlot.Set(ATTESTATIONS_FAMILY, fmt.Sprintf("%d", attestedSlot), gcp_bigtable.Timestamp((MAX_BLOCK_NUMBER-inclusionSlot)*1000), []byte{}) + mutInclusionSlot.Set(ATTESTATIONS_FAMILY, fmt.Sprintf("%d", attestedSlot), gcp_bigtable.Timestamp((MAX_CL_BLOCK_NUMBER-inclusionSlot)*1000), []byte{}) key := fmt.Sprintf("%s:%s:%s:%s", bigtable.chainId, bigtable.validatorIndexToKey(uint64(validator)), ATTESTATIONS_FAMILY, bigtable.reversedPaddedEpoch(epoch)) mutsInclusionSlot = append(mutsInclusionSlot, mutInclusionSlot) @@ -732,7 +733,7 @@ func (bigtable *Bigtable) SaveProposals(blocks map[uint64]map[string]*types.Bloc continue } mut := gcp_bigtable.NewMutation() - mut.Set(PROPOSALS_FAMILY, "b", gcp_bigtable.Timestamp((MAX_BLOCK_NUMBER-b.Slot)*1000), []byte{}) + mut.Set(PROPOSALS_FAMILY, "b", gcp_bigtable.Timestamp((MAX_CL_BLOCK_NUMBER-b.Slot)*1000), []byte{}) key := fmt.Sprintf("%s:%s:%s:%s:%s", bigtable.chainId, bigtable.validatorIndexToKey(b.Proposer), PROPOSALS_FAMILY, bigtable.reversedPaddedEpoch(utils.EpochOfSlot(b.Slot)), bigtable.reversedPaddedSlot(b.Slot)) muts = append(muts, mut) @@ -789,7 +790,7 @@ func (bigtable *Bigtable) SaveSyncComitteeDuties(duties map[types.Slot]map[types for validator, participated := range validators { mut := gcp_bigtable.NewMutation() if participated { - mut.Set(SYNC_COMMITTEES_FAMILY, "s", gcp_bigtable.Timestamp((MAX_BLOCK_NUMBER-slot)*1000), []byte{}) + mut.Set(SYNC_COMMITTEES_FAMILY, "s", gcp_bigtable.Timestamp((MAX_CL_BLOCK_NUMBER-slot)*1000), []byte{}) participation++ } else { mut.Set(SYNC_COMMITTEES_FAMILY, "s", gcp_bigtable.Timestamp(0), []byte{}) @@ -1017,10 +1018,10 @@ func (bigtable *Bigtable) GetValidatorAttestationHistory(validators []uint64, st logger.Errorf("error parsing slot from row key %v: %v", r.Key(), err) return false } - inclusionSlot := MAX_BLOCK_NUMBER - uint64(ri.Timestamp)/1000 + inclusionSlot := MAX_CL_BLOCK_NUMBER - uint64(ri.Timestamp)/1000 status := uint64(1) - if inclusionSlot == MAX_BLOCK_NUMBER { + if inclusionSlot == MAX_CL_BLOCK_NUMBER { inclusionSlot = 0 status = 0 } @@ -1218,7 +1219,7 @@ func (bigtable *Bigtable) GetSyncParticipationBySlotRange(startSlot, endSlot uin logger.Errorf("error parsing epoch from row key %v: %v", r.Key(), err) return false } - slot = MAX_BLOCK_NUMBER - slot + slot = MAX_CL_BLOCK_NUMBER - slot for _, ri := range r[SYNC_COMMITTEES_PARTICIPATION_FAMILY] { res[slot] = binary.LittleEndian.Uint64(ri.Value) @@ -1307,10 +1308,10 @@ func (bigtable *Bigtable) GetValidatorMissedAttestationHistory(validators []uint return false } - inclusionSlot := MAX_BLOCK_NUMBER - uint64(ri.Timestamp)/1000 + inclusionSlot := MAX_CL_BLOCK_NUMBER - uint64(ri.Timestamp)/1000 status := uint64(1) - if inclusionSlot == MAX_BLOCK_NUMBER { + if inclusionSlot == MAX_CL_BLOCK_NUMBER { status = 0 } @@ -1406,13 +1407,13 @@ func (bigtable *Bigtable) GetValidatorSyncDutiesHistory(validators []uint64, sta logger.Errorf("error parsing slot from row key %v: %v", r.Key(), err) return false } - slot = MAX_BLOCK_NUMBER - slot + slot = MAX_CL_BLOCK_NUMBER - slot for _, ri := range r[SYNC_COMMITTEES_FAMILY] { - inclusionSlot := MAX_BLOCK_NUMBER - uint64(ri.Timestamp)/1000 + inclusionSlot := MAX_CL_BLOCK_NUMBER - uint64(ri.Timestamp)/1000 status := uint64(1) // 1: participated - if inclusionSlot == MAX_BLOCK_NUMBER { + if inclusionSlot == MAX_CL_BLOCK_NUMBER { inclusionSlot = 0 status = 0 // 0: missed } @@ -1718,11 +1719,11 @@ func (bigtable *Bigtable) GetValidatorProposalHistory(validators []uint64, start logger.Errorf("error parsing slot from row key %v: %v", r.Key(), err) return false } - proposalSlot = MAX_BLOCK_NUMBER - proposalSlot - inclusionSlot := MAX_BLOCK_NUMBER - uint64(r[PROPOSALS_FAMILY][0].Timestamp)/1000 + proposalSlot = MAX_CL_BLOCK_NUMBER - proposalSlot + inclusionSlot := MAX_CL_BLOCK_NUMBER - uint64(r[PROPOSALS_FAMILY][0].Timestamp)/1000 status := uint64(1) - if inclusionSlot == MAX_BLOCK_NUMBER { + if inclusionSlot == MAX_CL_BLOCK_NUMBER { inclusionSlot = 0 status = 2 } @@ -2166,5 +2167,5 @@ func (bigtable *Bigtable) reversedPaddedEpoch(epoch uint64) string { } func (bigtable *Bigtable) reversedPaddedSlot(slot uint64) string { - return fmt.Sprintf("%09d", MAX_BLOCK_NUMBER-slot) + return fmt.Sprintf("%09d", MAX_CL_BLOCK_NUMBER-slot) } diff --git a/db/bigtable_eth1.go b/db/bigtable_eth1.go index 849af75237..5d9fac61f7 100644 --- a/db/bigtable_eth1.go +++ b/db/bigtable_eth1.go @@ -184,7 +184,7 @@ func (bigtable *Bigtable) CheckForGapsInBlocksTable(lookback int) (gapFound bool logger.Errorf("error parsing block number from key %v: %v", r.Key(), err) return false } - c = MAX_BLOCK_NUMBER - c + c = MAX_EL_BLOCK_NUMBER - c if c%10000 == 0 { logger.Infof("scanning, currently at block %v", c) @@ -266,7 +266,7 @@ func (bigtable *Bigtable) CheckForGapsInDataTable(lookback int) error { logger.Errorf("error parsing block number from key %v: %v", r.Key(), err) return false } - c = MAX_BLOCK_NUMBER - c + c = MAX_EL_BLOCK_NUMBER - c if c%10000 == 0 { logger.Infof("scanning, currently at block %v", c) @@ -338,7 +338,7 @@ func (bigtable *Bigtable) getLastBlockInDataTableFromBigtable() (int, error) { logger.Errorf("error parsing block number from key %v: %v", r.Key(), err) return false } - c = MAX_BLOCK_NUMBER - c + c = MAX_EL_BLOCK_NUMBER - c lastBlock = c return c == 0 // required as the block with number 0 will be returned as first block before the most recent one @@ -365,7 +365,7 @@ func (bigtable *Bigtable) getLastBlockInBlocksTableFromBigtable() (int, error) { logger.Errorf("error parsing block number from key %v: %v", r.Key(), err) return false } - c = MAX_BLOCK_NUMBER - c + c = MAX_EL_BLOCK_NUMBER - c lastBlock = c return c == 0 // required as the block with number 0 will be returned as first block before the most recent one @@ -409,7 +409,7 @@ func (bigtable *Bigtable) GetMostRecentBlockFromDataTable() (*types.Eth1BlockInd return false } - c = MAX_BLOCK_NUMBER - c + c = MAX_EL_BLOCK_NUMBER - c err = proto.Unmarshal(row[DEFAULT_FAMILY][0].Value, &block) if err != nil { @@ -596,7 +596,7 @@ func (bigtable *Bigtable) GetBlocksDescending(start, limit uint64) ([]*types.Eth } func reversedPaddedBlockNumber(blockNumber uint64) string { - return fmt.Sprintf("%09d", MAX_BLOCK_NUMBER-blockNumber) + return fmt.Sprintf("%09d", MAX_EL_BLOCK_NUMBER-blockNumber) } func reversePaddedBigtableTimestamp(timestamp *timestamppb.Timestamp) string {