diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index be1dfa79da..36eb91f84d 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -14,14 +14,17 @@ jobs: name: Build runs-on: ubuntu-latest steps: - - name: Set up Go 1.18 - uses: actions/setup-go@v3 + - name: Check out code into the Go module directory + uses: actions/checkout@v3 + + - name: Set up Go 1.20 + uses: actions/setup-go@v4 with: - go-version: 1.18 + go-version-file: './go.mod' id: go - - name: Check out code into the Go module directory - uses: actions/checkout@v3 + - name: Output go version + run: go version - name: Check gofmt run: | diff --git a/.github/workflows/staticcheck.yml b/.github/workflows/staticcheck.yml index 10f7af1b21..2a985cd67f 100644 --- a/.github/workflows/staticcheck.yml +++ b/.github/workflows/staticcheck.yml @@ -17,7 +17,7 @@ jobs: fail-fast: false matrix: os: ["ubuntu-latest"] - go: ["1.18.x"] + go: ["1.20.x"] runs-on: ${{ matrix.os }} steps: - uses: actions/checkout@v1 @@ -31,6 +31,6 @@ jobs: - run: "go vet ./..." - uses: dominikh/staticcheck-action@v1.2.0 with: - version: "2022.1.1" + version: "2023.1.5" install-go: false cache-key: ${{ matrix.go }} \ No newline at end of file diff --git a/.gitignore b/.gitignore index 122ec2e3d2..49ff779018 100644 --- a/.gitignore +++ b/.gitignore @@ -29,3 +29,4 @@ lastAttestationCache cmd/forkwatch **/.DS_Store _gitignore/ +local-deployment/config.yml \ No newline at end of file diff --git a/Makefile b/Makefile index ac09020931..ff64f35994 100644 --- a/Makefile +++ b/Makefile @@ -5,7 +5,7 @@ BUILDDATE=`date -u +"%Y-%m-%dT%H:%M:%S%:z"` PACKAGE=eth2-exporter LDFLAGS="-X ${PACKAGE}/version.Version=${VERSION} -X ${PACKAGE}/version.BuildDate=${BUILDDATE} -X ${PACKAGE}/version.GitCommit=${GITCOMMIT} -X ${PACKAGE}/version.GitDate=${GITDATE} -s -w" -all: explorer stats frontend-data-updater eth1indexer blobindexer ethstore-exporter rewards-exporter node-jobs-processor signatures notification-sender notification-collector +all: explorer stats frontend-data-updater eth1indexer blobindexer ethstore-exporter rewards-exporter node-jobs-processor signatures notification-sender notification-collector misc lint: golint ./... diff --git a/README.md b/README.md index d3c5dabdde..07068e7e3c 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ The explorer provides a comprehensive and easy to use interface for the upcoming Ethereum beacon chain. It makes it easy to view proposed blocks, follow attestations and monitor your staking activity. -[![Discord](https://img.shields.io/badge/Discord-%235865F2.svg?style=for-the-badge&logo=discord&logoColor=white)](https://discord.io/beaconchain) +[![Discord](https://img.shields.io/badge/Discord-%235865F2.svg?style=for-the-badge&logo=discord&logoColor=white)](https://dsc.gg/beaconchain) [![Twitter](https://img.shields.io/badge/Twitter-%231DA1F2.svg?style=for-the-badge&logo=Twitter&logoColor=white)](https://twitter.com/beaconcha_in) [![Badge](https://github.com/gobitfly/eth2-beaconchain-explorer/workflows/Build/badge.svg)](https://github.com/gobitfly/eth2-beaconchain-explorer/actions?query=workflow%3A%22Build+%26+Publish+Docker+images%22) diff --git a/bigtable_config.md b/bigtable_config.md index cc850bf59e..f5f4e6702a 100644 --- a/bigtable_config.md +++ b/bigtable_config.md @@ -2,52 +2,101 @@ This document summarized the bigtable configuration options and table definitions required to run the beaconcha.in explorer. All settings can be applied either by using the GCP bigtable web interface or the `cbt` tool. ---- -Table name: `beaconchain` +Table name: `beaconchain_validator_history` + +``` +cbt -project $PROJECT -instance $INSTANCE createtable beaconchain_validator_history +``` Column families: -* Name: `at` | GC Policy: Version based policy with a maximum of 1 versions +* Name: `vb` | GC Policy: None +* Name: `ha` | GC Policy: None +* Name: `at` | GC Policy: None +* Name: `pr` | GC Policy: None +* Name: `sc` | GC Policy: None +* Name: `sp` | GC Policy: None * Name: `id` | GC Policy: None -* Name: `pr` | GC Policy: Version based policy with a maximum of 1 versions -* Name: `sc` | GC Policy: Version based policy with a maximum of 1 versions * Name: `stats` | GC Policy: None -* Name: `vb` | GC Policy: None +``` +cbt -project $PROJECT -instance $INSTANCE createfamily beaconchain_validator_history vb +cbt -project $PROJECT -instance $INSTANCE createfamily beaconchain_validator_history ha +cbt -project $PROJECT -instance $INSTANCE createfamily beaconchain_validator_history at +cbt -project $PROJECT -instance $INSTANCE createfamily beaconchain_validator_history pr +cbt -project $PROJECT -instance $INSTANCE createfamily beaconchain_validator_history sc +cbt -project $PROJECT -instance $INSTANCE createfamily beaconchain_validator_history sp +cbt -project $PROJECT -instance $INSTANCE createfamily beaconchain_validator_history id +cbt -project $PROJECT -instance $INSTANCE createfamily beaconchain_validator_history stats +``` ---- Table name: `beaconchain_validators` +``` +cbt -project $PROJECT -instance $INSTANCE createtable beaconchain_validators +``` + Column families: * Name: `at` | GC Policy: Version based policy with a maximum of 1 versions +``` +cbt -project $PROJECT -instance $INSTANCE createfamily beaconchain_validators at + +cbt -project $PROJECT -instance $INSTANCE setgcpolicy beaconchain_validators at maxversions=1 +``` ---- Table name: `blocks` +``` +cbt -project $PROJECT -instance $INSTANCE createtable blocks +``` + Column families: * Name: `default` | GC Policy: Version based policy with a maximum of 1 versions ----- -Table name: `cache` - -Column families: -* Name: `10_min` | GC Policy: Version based policy with a maximum of 1 versions and a maximum age of 10 minutes -* Name: `1_day` | GC Policy: Version based policy with a maximum of 1 versions and a maximum age of 1 day -* Name: `1_hour` | GC Policy: Version based policy with a maximum of 1 versions and a maximum age of 1 hour +``` +cbt -project $PROJECT -instance $INSTANCE createfamily blocks default +cbt -project $PROJECT -instance $INSTANCE setgcpolicy blocks default maxversions=1 +``` ---- Table name: `data` +``` +cbt -project $PROJECT -instance $INSTANCE createtable data +``` + Column families: * Name: `c` | GC Policy: Age based policy with a max age of 1 day * Name: `f` | GC Policy: None +``` +cbt -project $PROJECT -instance $INSTANCE createfamily data c +cbt -project $PROJECT -instance $INSTANCE createfamily data f + +cbt -project $PROJECT -instance $INSTANCE setgcpolicy data c maxage=1d +``` ---- Table name: `machine_metrics` +``` +cbt -project $PROJECT -instance $INSTANCE createtable machine_metrics +``` + Column families: * Name: `mm` | GC Policy: Age based policy with a max age of 31 days +``` +cbt -project $PROJECT -instance $INSTANCE createfamily machine_metrics mm + +cbt -project $PROJECT -instance $INSTANCE setgcpolicy machine_metrics mm maxage=31d +``` ---- Table name: `metadata` +``` +cbt -project $PROJECT -instance $INSTANCE createtable metadata +``` + Column families: * Name: `a` | GC Policy: None * Name: `c` | GC Policy: None @@ -56,9 +105,30 @@ Column families: * Name: `erc721` | GC Policy: None * Name: `series` | GC Policy: Version based policy with a maximum of 1 versions +``` +cbt -project $PROJECT -instance $INSTANCE createfamily metadata a +cbt -project $PROJECT -instance $INSTANCE createfamily metadata c +cbt -project $PROJECT -instance $INSTANCE createfamily metadata erc1155 +cbt -project $PROJECT -instance $INSTANCE createfamily metadata erc20 +cbt -project $PROJECT -instance $INSTANCE createfamily metadata erc721 +cbt -project $PROJECT -instance $INSTANCE createfamily metadata series + +cbt -project $PROJECT -instance $INSTANCE setgcpolicy metadata series maxversions=1 +``` ---- Table name: `metadata_updates` +``` +cbt -project $PROJECT -instance $INSTANCE createtable metadata_updates +``` + Column families: * Name: `blocks` | GC Policy: Age based policy with a max age of 1 day -* Name: `f` | GC Policy: None \ No newline at end of file +* Name: `f` | GC Policy: None + +``` +cbt -project $PROJECT -instance $INSTANCE createfamily metadata_updates blocks +cbt -project $PROJECT -instance $INSTANCE createfamily metadata_updates f + +cbt -project $PROJECT -instance $INSTANCE setgcpolicy metadata_updates blocks maxage=1d +``` \ No newline at end of file diff --git a/cache/bigtable_cache.go b/cache/bigtable_cache.go deleted file mode 100644 index 7f45d56320..0000000000 --- a/cache/bigtable_cache.go +++ /dev/null @@ -1,204 +0,0 @@ -package cache - -import ( - "context" - "encoding/json" - "fmt" - "time" - - gcp_bigtable "cloud.google.com/go/bigtable" - "github.com/sirupsen/logrus" -) - -const ( - TABLE_CACHE = "cache" - FAMILY_TEN_MINUTES = "10_min" - FAMILY_ONE_HOUR = "1_hour" - FAMILY_ONE_DAY = "1_day" - COLUMN_DATA = "d" -) - -type BigtableCache struct { - client *gcp_bigtable.Client - - tableCache *gcp_bigtable.Table - - chainId string -} - -func InitBigtableCache(client *gcp_bigtable.Client, chainId string) *BigtableCache { - bt := &BigtableCache{ - client: client, - tableCache: client.Open(TABLE_CACHE), - chainId: chainId, - } - - return bt -} - -func (cache *BigtableCache) Set(ctx context.Context, key string, value any, expiration time.Duration) error { - - family := FAMILY_TEN_MINUTES - if expiration.Minutes() >= 60 { - family = FAMILY_ONE_HOUR - } - if expiration.Hours() > 1 { - family = FAMILY_ONE_DAY - } - - valueMarshal, err := json.Marshal(value) - if err != nil { - return err - } - - ts := gcp_bigtable.Now() - mut := gcp_bigtable.NewMutation() - mut.Set(family, COLUMN_DATA, ts, valueMarshal) - - err = cache.tableCache.Apply(ctx, fmt.Sprintf("C:%s", key), mut) - if err != nil { - return err - } - return nil -} - -func (cache *BigtableCache) setByte(ctx context.Context, key string, value []byte, expiration time.Duration) error { - - family := FAMILY_TEN_MINUTES - if expiration.Minutes() >= 60 { - family = FAMILY_ONE_HOUR - } - if expiration.Hours() > 1 { - family = FAMILY_ONE_DAY - } - - ts := gcp_bigtable.Now() - mut := gcp_bigtable.NewMutation() - mut.Set(family, COLUMN_DATA, ts, value) - - err := cache.tableCache.Apply(ctx, fmt.Sprintf("C:%s", key), mut) - if err != nil { - return err - } - return nil -} - -func (cache *BigtableCache) SetString(ctx context.Context, key, value string, expiration time.Duration) error { - return cache.setByte(ctx, key, []byte(value), expiration) -} - -func (cache *BigtableCache) SetUint64(ctx context.Context, key string, value uint64, expiration time.Duration) error { - return cache.setByte(ctx, key, ui64tob(value), expiration) -} - -func (cache *BigtableCache) SetBool(ctx context.Context, key string, value bool, expiration time.Duration) error { - return cache.setByte(ctx, key, booltob(value), expiration) -} - -func (cache *BigtableCache) Get(ctx context.Context, key string, returnValue any) (any, error) { - res, err := cache.getByte(ctx, key) - if err != nil { - return nil, err - } - - err = json.Unmarshal([]byte(res), returnValue) - if err != nil { - // cache.remoteRedisCache.Del(ctx, key).Err() - logrus.Errorf("error (bigtable_cache.go / Get) unmarshalling data for key %v: %v", key, err) - return nil, err - } - - return returnValue, nil -} - -func (cache *BigtableCache) getByte(ctx context.Context, key string) ([]byte, error) { - filter := gcp_bigtable.ChainFilters( - gcp_bigtable.ColumnFilter("d"), - gcp_bigtable.LatestNFilter(1), - ) - - row, err := cache.tableCache.ReadRow(ctx, fmt.Sprintf("C:%s", key), gcp_bigtable.RowFilter(filter)) - if err != nil { - return nil, err - } - - if len(row) == 0 { - return nil, fmt.Errorf("error getting key: %s no result available, row: %+v", key, row) - } - - // iterate over all column families and only take the most recent entry - res := new(gcp_bigtable.ReadItem) - for _, column := range row { - if len(column) != 1 { - return nil, fmt.Errorf("error unexpected number of results returned key: %s no result available, row: %+v", key, row) - } - if res == nil { - res = &column[0] - } - - if res.Timestamp.Time().Before(column[0].Timestamp.Time()) { - res = &column[0] - } - } - - return res.Value, nil -} - -func (cache *BigtableCache) GetString(ctx context.Context, key string) (string, error) { - - res, err := cache.getByte(ctx, key) - if err != nil { - return "", err - } - return string(res), nil -} - -func (cache *BigtableCache) GetUint64(ctx context.Context, key string) (uint64, error) { - - res, err := cache.getByte(ctx, key) - if err != nil { - return 0, err - } - - return btoi64(res), nil -} - -func (cache *BigtableCache) GetBool(ctx context.Context, key string) (bool, error) { - - res, err := cache.getByte(ctx, key) - if err != nil { - return false, err - } - - return btobool(res), nil -} - -func ui64tob(val uint64) []byte { - r := make([]byte, 8) - for i := uint64(0); i < 8; i++ { - r[i] = byte((val >> (i * 8)) & 0xff) - } - return r -} - -func btoi64(val []byte) uint64 { - r := uint64(0) - for i := uint64(0); i < 8; i++ { - r |= uint64(val[i]) << (8 * i) - } - return r -} - -func booltob(val bool) []byte { - r := make([]byte, 1) - if val { - r[0] = 1 - } else { - r[0] = 0 - } - return r -} - -func btobool(val []byte) bool { - return val[0] == 1 -} diff --git a/cache/tiered_cache.go b/cache/tiered_cache.go index 007cecb650..531c7bf837 100644 --- a/cache/tiered_cache.go +++ b/cache/tiered_cache.go @@ -8,7 +8,6 @@ import ( "strconv" "time" - gcp_bigtable "cloud.google.com/go/bigtable" "github.com/coocood/freecache" "github.com/sirupsen/logrus" ) @@ -48,18 +47,6 @@ func MustInitTieredCache(redisAddress string) { } } -func MustInitTieredCacheBigtable(client *gcp_bigtable.Client, chainId string) { - localCache := freecache.NewCache(100 * 1024 * 1024) // 100 MB - - cache := InitBigtableCache(client, chainId) - - TieredCache = &tieredCache{ - remoteCache: cache, - localGoCache: localCache, - } - -} - func (cache *tieredCache) SetString(key, value string, expiration time.Duration) error { ctx, cancel := context.WithTimeout(context.Background(), time.Second*30) defer cancel() diff --git a/cmd/bundle/main.go b/cmd/bundle/main.go index 87fd5bdce3..0c5e76ad25 100644 --- a/cmd/bundle/main.go +++ b/cmd/bundle/main.go @@ -4,7 +4,6 @@ import ( "crypto/md5" "eth2-exporter/utils" "fmt" - "io/ioutil" "log" "os" "path" @@ -72,7 +71,7 @@ func bundle(staticDir string) (map[string]string, error) { } for _, match := range matches { - code, err := ioutil.ReadFile(match) + code, err := os.ReadFile(match) if err != nil { return nameMapping, fmt.Errorf("error reading file %v", err) } @@ -97,7 +96,7 @@ func bundle(staticDir string) (map[string]string, error) { newPath := strings.ReplaceAll(matchHash, "static/", "") nameMapping[path] = newPath - err = ioutil.WriteFile(matchHash, code, 0755) + err = os.WriteFile(matchHash, code, 0755) if err != nil { return nameMapping, fmt.Errorf("error failed to write file %v", err) } @@ -116,7 +115,7 @@ func replaceFilesNames(files map[string]string) error { return err } for _, match := range matches { - html, err := ioutil.ReadFile(match) + html, err := os.ReadFile(match) if err != nil { return err } @@ -125,7 +124,7 @@ func replaceFilesNames(files map[string]string) error { // logrus.Info("replacing: ", oldPath, " with: ", newPath) h = strings.ReplaceAll(h, oldPath, newPath) } - err = ioutil.WriteFile(match, []byte(h), 0755) + err = os.WriteFile(match, []byte(h), 0755) if err != nil { return err } diff --git a/cmd/eth1indexer/main.go b/cmd/eth1indexer/main.go index e4b8b9fd0f..eeab6fb2da 100644 --- a/cmd/eth1indexer/main.go +++ b/cmd/eth1indexer/main.go @@ -14,9 +14,10 @@ import ( "eth2-exporter/version" "flag" "fmt" - "io/ioutil" + "io" "math/big" "net/http" + "os" "strconv" "strings" "sync/atomic" @@ -45,6 +46,7 @@ func main() { offsetBlocks := flag.Int64("blocks.offset", 100, "Blocks offset") checkBlocksGaps := flag.Bool("blocks.gaps", false, "Check for gaps in the blocks table") checkBlocksGapsLookback := flag.Int("blocks.gaps.lookback", 1000000, "Lookback for gaps check of the blocks table") + traceMode := flag.String("blocks.tracemode", "parity/geth", "Trace mode to use, can bei either 'parity', 'geth' or 'parity/geth' for both") concurrencyData := flag.Int64("data.concurrency", 30, "Concurrency to use when indexing data from bigtable") startData := flag.Int64("data.start", 0, "Block to start indexing") @@ -112,7 +114,15 @@ func main() { defer db.WriterDb.Close() if erigonEndpoint == nil || *erigonEndpoint == "" { - utils.LogFatal(nil, "no erigon node url provided", 0) + + if utils.Config.Eth1ErigonEndpoint == "" { + + utils.LogFatal(nil, "no erigon node url provided", 0) + } else { + logrus.Info("applying erigon endpoint from config") + *erigonEndpoint = utils.Config.Eth1ErigonEndpoint + } + } logrus.Infof("using erigon node at %v", *erigonEndpoint) @@ -208,7 +218,7 @@ func main() { cache := freecache.NewCache(100 * 1024 * 1024) // 100 MB limit if *block != 0 { - err = IndexFromNode(bt, client, *block, *block, *concurrencyBlocks) + err = IndexFromNode(bt, client, *block, *block, *concurrencyBlocks, *traceMode) if err != nil { logrus.WithError(err).Fatalf("error indexing from node, start: %v end: %v concurrency: %v", *block, *block, *concurrencyBlocks) } @@ -233,7 +243,7 @@ func main() { } if *endBlocks != 0 && *startBlocks < *endBlocks { - err = IndexFromNode(bt, client, *startBlocks, *endBlocks, *concurrencyBlocks) + err = IndexFromNode(bt, client, *startBlocks, *endBlocks, *concurrencyBlocks, *traceMode) if err != nil { logrus.WithError(err).Fatalf("error indexing from node, start: %v end: %v concurrency: %v", *startBlocks, *endBlocks, *concurrencyBlocks) } @@ -303,7 +313,7 @@ func main() { endBlock = int64(lastBlockFromNode) } - err = IndexFromNode(bt, client, startBlock, endBlock, *concurrencyBlocks) + err = IndexFromNode(bt, client, startBlock, endBlock, *concurrencyBlocks, *traceMode) if err != nil { errMsg := "error indexing from node" errFields := map[string]interface{}{ @@ -384,7 +394,7 @@ func main() { func UpdateTokenPrices(bt *db.Bigtable, client *rpc.ErigonClient, tokenListPath string) error { - tokenListContent, err := ioutil.ReadFile(tokenListPath) + tokenListContent, err := os.ReadFile(tokenListPath) if err != nil { return err } @@ -424,7 +434,7 @@ func UpdateTokenPrices(bt *db.Bigtable, client *rpc.ErigonClient, tokenListPath return fmt.Errorf("error querying defillama api: %v", resp.Status) } - body, err := ioutil.ReadAll(resp.Body) + body, err := io.ReadAll(resp.Body) if err != nil { return err } @@ -715,7 +725,7 @@ func ProcessMetadataUpdates(bt *db.Bigtable, client *rpc.ErigonClient, prefix st // } } -func IndexFromNode(bt *db.Bigtable, client *rpc.ErigonClient, start, end, concurrency int64) error { +func IndexFromNode(bt *db.Bigtable, client *rpc.ErigonClient, start, end, concurrency int64, traceMode string) error { ctx := context.Background() g, gCtx := errgroup.WithContext(ctx) g.SetLimit(int(concurrency)) @@ -736,7 +746,7 @@ func IndexFromNode(bt *db.Bigtable, client *rpc.ErigonClient, start, end, concur } blockStartTs := time.Now() - bc, timings, err := client.GetBlock(i) + bc, timings, err := client.GetBlock(i, traceMode) if err != nil { return fmt.Errorf("error getting block: %v from ethereum node err: %w", i, err) } @@ -797,7 +807,7 @@ func ImportMainnetERC20TokenMetadataFromTokenDirectory(bt *db.Bigtable) { utils.LogFatal(err, "getting client error", 0) } - body, err := ioutil.ReadAll(resp.Body) + body, err := io.ReadAll(resp.Body) if err != nil { utils.LogFatal(err, "reading body for ERC20 tokens error", 0) @@ -850,7 +860,7 @@ func ImportMainnetERC20TokenMetadataFromTokenDirectory(bt *db.Bigtable) { resp, err := client.Get(token.LogoURI) if err == nil && resp.StatusCode == 200 { - body, err := ioutil.ReadAll(resp.Body) + body, err := io.ReadAll(resp.Body) if err != nil { utils.LogFatal(err, "reading body for ERC20 token logo URI error", 0) @@ -880,7 +890,7 @@ func ImportNameLabels(bt *db.Bigtable) { res := make(map[string]*NameEntry) - data, err := ioutil.ReadFile("") + data, err := os.ReadFile("") if err != nil { utils.LogFatal(err, "reading file error", 0) diff --git a/cmd/ethstore-exporter/main.go b/cmd/ethstore-exporter/main.go index 5dec0b30ee..5581117c31 100644 --- a/cmd/ethstore-exporter/main.go +++ b/cmd/ethstore-exporter/main.go @@ -8,6 +8,8 @@ import ( "eth2-exporter/version" "flag" "fmt" + "strconv" + "strings" _ "github.com/jackc/pgx/v4/stdlib" "github.com/sirupsen/logrus" @@ -21,6 +23,8 @@ func main() { errorInterval := flag.Duration("error-intv", 0, "Error interval") sleepInterval := flag.Duration("sleep-intv", 0, "Sleep interval") versionFlag := flag.Bool("version", false, "Show version and exit") + dayToReexport := flag.Int64("day", -1, "Day to reexport") + daysToReexport := flag.String("days", "", "Days to reexport") flag.Parse() if *versionFlag { @@ -56,6 +60,27 @@ func main() { defer db.ReaderDb.Close() defer db.WriterDb.Close() - exporter.StartEthStoreExporter(*bnAddress, *enAddress, *updateInterval, *errorInterval, *sleepInterval) + var startDayReexport int64 = -1 + var endDayReexport int64 = -1 + + if *daysToReexport != "" { + s := strings.Split(*daysToReexport, "-") + if len(s) < 2 { + utils.LogFatal(nil, fmt.Sprintf("invalid 'days' flag: %s, expected something of the form 'startDay-endDay'", *daysToReexport), 0) + } + startDayReexport, err = strconv.ParseInt(s[0], 10, 64) + if err != nil { + utils.LogFatal(err, "error parsing first day of daysToExport flag to int", 0) + } + endDayReexport, err = strconv.ParseInt(s[1], 10, 64) + if err != nil { + utils.LogFatal(err, "error parsing last day of daysToExport flag to int", 0) + } + } else if *dayToReexport >= 0 { + startDayReexport = *dayToReexport + endDayReexport = *dayToReexport + } + + exporter.StartEthStoreExporter(*bnAddress, *enAddress, *updateInterval, *errorInterval, *sleepInterval, startDayReexport, endDayReexport) logrus.Println("exiting...") } diff --git a/cmd/explorer/main.go b/cmd/explorer/main.go index c777059be1..ab0b6c4ca8 100644 --- a/cmd/explorer/main.go +++ b/cmd/explorer/main.go @@ -80,11 +80,6 @@ func main() { utils.LogFatal(err, "invalid chain configuration specified, you must specify the slots per epoch, seconds per slot and genesis timestamp in the config file", 0) } - err = handlers.CheckAndPreloadImprint() - if err != nil { - logrus.Fatalf("error check / preload imprint: %v", err) - } - if utils.Config.Pprof.Enabled { go func() { logrus.Infof("starting pprof http server on port %s", utils.Config.Pprof.Port) @@ -190,13 +185,9 @@ func main() { } wg.Wait() - if utils.Config.TieredCacheProvider == "bigtable" && len(utils.Config.RedisCacheEndpoint) == 0 { - cache.MustInitTieredCacheBigtable(db.BigtableClient.GetClient(), fmt.Sprintf("%d", utils.Config.Chain.ClConfig.DepositChainID)) - logrus.Infof("tiered Cache initialized, latest finalized epoch: %v", services.LatestFinalizedEpoch()) - } - if utils.Config.TieredCacheProvider != "bigtable" && utils.Config.TieredCacheProvider != "redis" { - logrus.Fatalf("no cache provider set, please set TierdCacheProvider (example redis, bigtable)") + if utils.Config.TieredCacheProvider != "redis" { + logrus.Fatalf("no cache provider set, please set TierdCacheProvider (example redis)") } defer db.ReaderDb.Close() @@ -326,6 +317,7 @@ func main() { apiV1Router.HandleFunc("/validator/eth1/{address}", handlers.ApiValidatorByEth1Address).Methods("GET", "OPTIONS") apiV1Router.HandleFunc("/validator/withdrawalCredentials/{withdrawalCredentialsOrEth1address}", handlers.ApiWithdrawalCredentialsValidators).Methods("GET", "OPTIONS") apiV1Router.HandleFunc("/validators/queue", handlers.ApiValidatorQueue).Methods("GET", "OPTIONS") + apiV1Router.HandleFunc("/validators/proposalLuck", handlers.ApiProposalLuck).Methods("GET", "OPTIONS") apiV1Router.HandleFunc("/graffitiwall", handlers.ApiGraffitiwall).Methods("GET", "OPTIONS") apiV1Router.HandleFunc("/chart/{chart}", handlers.ApiChart).Methods("GET", "OPTIONS") apiV1Router.HandleFunc("/user/token", handlers.APIGetToken).Methods("POST", "OPTIONS") @@ -475,7 +467,7 @@ func main() { router.HandleFunc("/validator/{pubkey}/deposits", handlers.ValidatorDeposits).Methods("GET") router.HandleFunc("/validator/{index}/slashings", handlers.ValidatorSlashings).Methods("GET") router.HandleFunc("/validator/{index}/effectiveness", handlers.ValidatorAttestationInclusionEffectiveness).Methods("GET") - router.HandleFunc("/validator/{pubkey}/save", handlers.ValidatorSave).Methods("POST") + router.HandleFunc("/validator/{pubkey}/name", handlers.SaveValidatorName).Methods("POST") router.HandleFunc("/watchlist/add", handlers.UsersModalAddValidator).Methods("POST") router.HandleFunc("/validator/{pubkey}/remove", handlers.UserValidatorWatchlistRemove).Methods("POST") router.HandleFunc("/validator/{index}/stats", handlers.ValidatorStatsTable).Methods("GET") @@ -485,8 +477,6 @@ func main() { router.HandleFunc("/validators/slashings/data", handlers.ValidatorsSlashingsData).Methods("GET") router.HandleFunc("/validators/leaderboard", handlers.ValidatorsLeaderboard).Methods("GET") router.HandleFunc("/validators/leaderboard/data", handlers.ValidatorsLeaderboardData).Methods("GET") - router.HandleFunc("/validators/streakleaderboard", handlers.ValidatorsStreakLeaderboard).Methods("GET") - router.HandleFunc("/validators/streakleaderboard/data", handlers.ValidatorsStreakLeaderboardData).Methods("GET") router.HandleFunc("/validators/withdrawals", handlers.Withdrawals).Methods("GET") router.HandleFunc("/validators/withdrawals/data", handlers.WithdrawalsData).Methods("GET") router.HandleFunc("/validators/withdrawals/bls", handlers.BLSChangeData).Methods("GET") @@ -514,10 +504,7 @@ func main() { router.HandleFunc("/calculator", handlers.StakingCalculator).Methods("GET") router.HandleFunc("/search", handlers.Search).Methods("POST") router.HandleFunc("/search/{type}/{search}", handlers.SearchAhead).Methods("GET") - router.HandleFunc("/faq", handlers.Faq).Methods("GET") router.HandleFunc("/imprint", handlers.Imprint).Methods("GET") - router.HandleFunc("/poap", handlers.Poap).Methods("GET") - router.HandleFunc("/poap/data", handlers.PoapData).Methods("GET") router.HandleFunc("/mobile", handlers.MobilePage).Methods("GET") router.HandleFunc("/mobile", handlers.MobilePagePost).Methods("POST") router.HandleFunc("/tools/unitConverter", handlers.UnitConverter).Methods("GET") @@ -533,7 +520,6 @@ func main() { router.HandleFunc("/stakingServices", handlers.StakingServices).Methods("GET") - router.HandleFunc("/education", handlers.EducationServices).Methods("GET") router.HandleFunc("/ethClients", handlers.EthClientsServices).Methods("GET") router.HandleFunc("/pools", handlers.Pools).Methods("GET") router.HandleFunc("/relays", handlers.Relays).Methods("GET") @@ -578,7 +564,6 @@ func main() { oauthRouter := router.PathPrefix("/user").Subrouter() oauthRouter.HandleFunc("/authorize", handlers.UserAuthorizeConfirm).Methods("GET") - oauthRouter.HandleFunc("/cancel", handlers.UserAuthorizationCancel).Methods("GET") oauthRouter.Use(csrfHandler) authRouter := router.PathPrefix("/user").Subrouter() @@ -644,8 +629,6 @@ func main() { jsHandler := http.FileServer(http.Dir("static/js")) router.PathPrefix("/js").Handler(http.StripPrefix("/js/", jsHandler)) } - legalFs := http.Dir(utils.Config.Frontend.LegalDir) - router.PathPrefix("/legal").Handler(http.StripPrefix("/legal/", handlers.CustomFileServer(http.FileServer(legalFs), legalFs, handlers.NotFound))) fileSys := http.FS(static.Files) router.PathPrefix("/").Handler(handlers.CustomFileServer(http.FileServer(fileSys), fileSys, handlers.NotFound)) @@ -665,10 +648,10 @@ func main() { n.UseHandler(utils.SessionStore.SCS.LoadAndSave(router)) if utils.Config.Frontend.HttpWriteTimeout == 0 { - utils.Config.Frontend.HttpIdleTimeout = time.Second * 15 + utils.Config.Frontend.HttpWriteTimeout = time.Second * 15 } if utils.Config.Frontend.HttpReadTimeout == 0 { - utils.Config.Frontend.HttpIdleTimeout = time.Second * 15 + utils.Config.Frontend.HttpReadTimeout = time.Second * 15 } if utils.Config.Frontend.HttpIdleTimeout == 0 { utils.Config.Frontend.HttpIdleTimeout = time.Second * 60 diff --git a/cmd/frontend-data-updater/main.go b/cmd/frontend-data-updater/main.go index e2ee87f627..9431254675 100644 --- a/cmd/frontend-data-updater/main.go +++ b/cmd/frontend-data-updater/main.go @@ -3,12 +3,14 @@ package main import ( "eth2-exporter/cache" "eth2-exporter/db" + "eth2-exporter/rpc" "eth2-exporter/services" "eth2-exporter/types" "eth2-exporter/utils" "eth2-exporter/version" "flag" "fmt" + "math/big" _ "github.com/jackc/pgx/v4/stdlib" "github.com/sirupsen/logrus" @@ -70,14 +72,19 @@ func main() { if utils.Config.TieredCacheProvider == "redis" || len(utils.Config.RedisCacheEndpoint) != 0 { cache.MustInitTieredCache(utils.Config.RedisCacheEndpoint) - } else if utils.Config.TieredCacheProvider == "bigtable" && len(utils.Config.RedisCacheEndpoint) == 0 { - cache.MustInitTieredCacheBigtable(db.BigtableClient.GetClient(), fmt.Sprintf("%d", utils.Config.Chain.ClConfig.DepositChainID)) } - if utils.Config.TieredCacheProvider != "bigtable" && utils.Config.TieredCacheProvider != "redis" { + if utils.Config.TieredCacheProvider != "redis" { logrus.Fatalf("No cache provider set. Please set TierdCacheProvider (example redis, bigtable)") } + chainID := new(big.Int).SetUint64(utils.Config.Chain.ClConfig.DepositChainID) + rpcClient, err := rpc.NewLighthouseClient("http://"+cfg.Indexer.Node.Host+":"+cfg.Indexer.Node.Port, chainID) + if err != nil { + utils.LogFatal(err, "new explorer lighthouse client error", 0) + } + rpc.CurrentClient = rpcClient + logrus.Infof("initializing frontend services") services.Init() // Init frontend services logrus.Infof("frontend services initiated") diff --git a/cmd/migrations/bigtable/main.go b/cmd/migrations/bigtable/main.go index 69e24f936f..7653ccdd83 100644 --- a/cmd/migrations/bigtable/main.go +++ b/cmd/migrations/bigtable/main.go @@ -2,6 +2,7 @@ package main import ( "eth2-exporter/db" + "eth2-exporter/exporter" "eth2-exporter/rpc" "eth2-exporter/types" "eth2-exporter/utils" @@ -9,7 +10,6 @@ import ( "flag" "fmt" "math/big" - "strconv" "time" "github.com/sirupsen/logrus" @@ -22,6 +22,7 @@ func main() { start := flag.Uint64("start", 1, "Start epoch") end := flag.Uint64("end", 1, "End epoch") + concurrency := flag.Int("concurrency", 1, "Number of parallel epoch exports") versionFlag := flag.Bool("version", false, "Show version and exit") flag.Parse() @@ -48,6 +49,45 @@ func main() { } defer bt.Close() + db.MustInitDB(&types.DatabaseConfig{ + Username: cfg.WriterDatabase.Username, + Password: cfg.WriterDatabase.Password, + Name: cfg.WriterDatabase.Name, + Host: cfg.WriterDatabase.Host, + Port: cfg.WriterDatabase.Port, + MaxOpenConns: cfg.WriterDatabase.MaxOpenConns, + MaxIdleConns: cfg.WriterDatabase.MaxIdleConns, + }, &types.DatabaseConfig{ + Username: cfg.ReaderDatabase.Username, + Password: cfg.ReaderDatabase.Password, + Name: cfg.ReaderDatabase.Name, + Host: cfg.ReaderDatabase.Host, + Port: cfg.ReaderDatabase.Port, + MaxOpenConns: cfg.ReaderDatabase.MaxOpenConns, + MaxIdleConns: cfg.ReaderDatabase.MaxIdleConns, + }) + defer db.ReaderDb.Close() + defer db.WriterDb.Close() + db.MustInitFrontendDB(&types.DatabaseConfig{ + Username: cfg.Frontend.WriterDatabase.Username, + Password: cfg.Frontend.WriterDatabase.Password, + Name: cfg.Frontend.WriterDatabase.Name, + Host: cfg.Frontend.WriterDatabase.Host, + Port: cfg.Frontend.WriterDatabase.Port, + MaxOpenConns: cfg.Frontend.WriterDatabase.MaxOpenConns, + MaxIdleConns: cfg.Frontend.WriterDatabase.MaxIdleConns, + }, &types.DatabaseConfig{ + Username: cfg.Frontend.ReaderDatabase.Username, + Password: cfg.Frontend.ReaderDatabase.Password, + Name: cfg.Frontend.ReaderDatabase.Name, + Host: cfg.Frontend.ReaderDatabase.Host, + Port: cfg.Frontend.ReaderDatabase.Port, + MaxOpenConns: cfg.Frontend.ReaderDatabase.MaxOpenConns, + MaxIdleConns: cfg.Frontend.ReaderDatabase.MaxIdleConns, + }) + defer db.FrontendReaderDB.Close() + defer db.FrontendWriterDB.Close() + chainIDBig := new(big.Int).SetUint64(utils.Config.Chain.ClConfig.DepositChainID) rpcClient, err := rpc.NewLighthouseClient("http://"+cfg.Indexer.Node.Host+":"+cfg.Indexer.Node.Port, chainIDBig) @@ -55,84 +95,91 @@ func main() { utils.LogFatal(err, "new bigtable lighthouse client error", 0) } - for i := *start; i <= *end; i++ { - i := i - - logrus.Infof("exporting epoch %v", i) - - logrus.Infof("deleting existing epoch data") - err := bt.DeleteEpoch(i) - if err != nil { - utils.LogFatal(err, "deleting epoch error", 0) - } - - firstSlot := i * utils.Config.Chain.ClConfig.SlotsPerEpoch - lastSlot := (i+1)*utils.Config.Chain.ClConfig.SlotsPerEpoch - 1 + gOuter := errgroup.Group{} + gOuter.SetLimit(*concurrency) + for epoch := *start; epoch <= *end; epoch++ { + epoch := epoch + gOuter.Go(func() error { + logrus.Infof("exporting epoch %v", epoch) + start := time.Now() - c, err := rpcClient.GetSyncCommittee(fmt.Sprintf("%d", firstSlot), i) - if err != nil { - utils.LogFatal(err, "getting sync comittee error", 0) - } + startGetEpochData := time.Now() + logrus.Printf("retrieving data for epoch %v", epoch) - validatorsU64 := make([]uint64, len(c.Validators)) - for i, idxStr := range c.Validators { - idxU64, err := strconv.ParseUint(idxStr, 10, 64) + data, err := rpcClient.GetEpochData(epoch, false) if err != nil { - utils.LogFatal(err, "parsing validator index to uint error", 0) + logrus.Fatalf("error retrieving epoch data: %v", err) } - validatorsU64[i] = idxU64 - } - - logrus.Infof("saving sync assignments for %v validators", len(validatorsU64)) - - err = db.BigtableClient.SaveSyncCommitteesAssignments(firstSlot, lastSlot, validatorsU64) - if err != nil { - logrus.Fatalf("error saving sync committee assignments: %v", err) - } - logrus.Infof("exported sync committee assignments to bigtable in %v", i) - - data, err := rpcClient.GetEpochData(uint64(i), true) - if err != nil { - utils.LogFatal(err, "getting epoch data error", 0) - } - - g := new(errgroup.Group) - - g.Go(func() error { - return bt.SaveValidatorBalances(data.Epoch, data.Validators) - }) + logrus.WithFields(logrus.Fields{"duration": time.Since(startGetEpochData), "epoch": epoch}).Info("completed getting epoch-data") + logrus.Printf("data for epoch %v retrieved, took %v", epoch, time.Since(start)) - g.Go(func() error { - return bt.SaveAttestationAssignments(data.Epoch, data.ValidatorAssignmentes.AttestorAssignments) - }) - - g.Go(func() error { - return bt.SaveProposalAssignments(data.Epoch, data.ValidatorAssignmentes.ProposerAssignments) - }) - - g.Go(func() error { - return bt.SaveAttestations(data.Blocks) - }) + if len(data.Validators) == 0 { + logrus.Fatal("error retrieving epoch data: no validators received for epoch") + } - g.Go(func() error { - return bt.SaveProposals(data.Blocks) - }) + // export epoch data to bigtable + g := new(errgroup.Group) + g.SetLimit(6) + g.Go(func() error { + err = db.BigtableClient.SaveValidatorBalances(epoch, data.Validators) + if err != nil { + return fmt.Errorf("error exporting validator balances to bigtable: %v", err) + } + return nil + }) + g.Go(func() error { + err = db.BigtableClient.SaveProposalAssignments(epoch, data.ValidatorAssignmentes.ProposerAssignments) + if err != nil { + return fmt.Errorf("error exporting proposal assignments to bigtable: %v", err) + } + return nil + }) + g.Go(func() error { + err = db.BigtableClient.SaveAttestationDuties(data.AttestationDuties) + if err != nil { + return fmt.Errorf("error exporting attestations to bigtable: %v", err) + } + return nil + }) + g.Go(func() error { + err = db.BigtableClient.SaveProposals(data.Blocks) + if err != nil { + return fmt.Errorf("error exporting proposals to bigtable: %v", err) + } + return nil + }) + g.Go(func() error { + err = db.BigtableClient.SaveSyncComitteeDuties(data.SyncDuties) + if err != nil { + return fmt.Errorf("error exporting sync committee duties to bigtable: %v", err) + } + return nil + }) + g.Go(func() error { + err = db.BigtableClient.MigrateIncomeDataV1V2Schema(epoch) + if err != nil { + return fmt.Errorf("error exporting sync committee duties to bigtable: %v", err) + } + return nil + }) - g.Go(func() error { - return bt.SaveSyncComitteeDuties(data.Blocks) + err = g.Wait() + if err != nil { + return fmt.Errorf("error during bigtable export: %w", err) + } + logrus.WithFields(logrus.Fields{"duration": time.Since(start), "epoch": epoch}).Info("completed exporting epoch") + return nil }) + } - err = g.Wait() - - if err != nil { - utils.LogFatal(err, "wait group error", 0) - } + err = gOuter.Wait() + if err != nil { + logrus.Fatalf("error during bigtable export: %v", err) } } func monitor(configPath string) { - cfg := &types.Config{} err := utils.ReadConfig(cfg, configPath) if err != nil { @@ -169,42 +216,7 @@ func monitor(configPath string) { for i := head.FinalizedEpoch; i <= head.HeadEpoch; i++ { logrus.Infof("exporting epoch %v", i) - data, err := rpcClient.GetEpochData(i, true) - if err != nil { - utils.LogFatal(err, "getting epoch data error", 0) - } - - g := new(errgroup.Group) - - g.Go(func() error { - return bt.SaveValidatorBalances(data.Epoch, data.Validators) - }) - - g.Go(func() error { - return bt.SaveAttestationAssignments(data.Epoch, data.ValidatorAssignmentes.AttestorAssignments) - }) - - g.Go(func() error { - return bt.SaveProposalAssignments(data.Epoch, data.ValidatorAssignmentes.ProposerAssignments) - }) - - g.Go(func() error { - return bt.SaveAttestations(data.Blocks) - }) - - g.Go(func() error { - return bt.SaveProposals(data.Blocks) - }) - - g.Go(func() error { - return bt.SaveSyncComitteeDuties(data.Blocks) - }) - - err = g.Wait() - - if err != nil { - utils.LogFatal(err, "wait group error", 0) - } + exporter.ExportEpoch(i, rpcClient) } current = head.HeadEpoch } diff --git a/cmd/misc/main.go b/cmd/misc/main.go index 3b0a1d7788..b84ee48b2a 100644 --- a/cmd/misc/main.go +++ b/cmd/misc/main.go @@ -50,7 +50,7 @@ var opts = struct { func main() { configPath := flag.String("config", "config/default.config.yml", "Path to the config file") - flag.StringVar(&opts.Command, "command", "", "command to run, available: updateAPIKey, applyDbSchema, epoch-export, debug-rewards, clear-bigtable, index-old-eth1-blocks, update-aggregation-bits, historic-prices-export, index-missing-blocks, export-epoch-missed-slots, migrate-last-attestation-slot-bigtable") + flag.StringVar(&opts.Command, "command", "", "command to run, available: updateAPIKey, applyDbSchema, initBigtableSchema, epoch-export, debug-rewards, clear-bigtable, index-old-eth1-blocks, update-aggregation-bits, historic-prices-export, index-missing-blocks, export-epoch-missed-slots, migrate-last-attestation-slot-bigtable, generate-config-from-testnet-stub, export-genesis-validators") flag.Uint64Var(&opts.StartEpoch, "start-epoch", 0, "start epoch") flag.Uint64Var(&opts.EndEpoch, "end-epoch", 0, "end epoch") flag.Uint64Var(&opts.User, "user", 0, "user id") @@ -160,6 +160,13 @@ func main() { logrus.WithError(err).Fatal("error applying db schema") } logrus.Infof("db schema applied successfully") + case "initBigtableSchema": + logrus.Infof("initializing bigtable schema") + err := db.InitBigtableSchema() + if err != nil { + logrus.WithError(err).Fatal("error initializing bigtable schema") + } + logrus.Infof("bigtable schema initialization completed") case "epoch-export": logrus.Infof("exporting epochs %v - %v", opts.StartEpoch, opts.EndEpoch) @@ -173,12 +180,16 @@ func main() { } case "export-epoch-missed-slots": logrus.Infof("exporting epochs with missed slots") + latestFinalizedEpoch, err := db.GetLatestFinalizedEpoch() + if err != nil { + utils.LogError(err, "error getting latest finalized epoch from db", 0) + } epochs := []uint64{} err = db.ReaderDb.Select(&epochs, ` WITH last_exported_epoch AS ( SELECT (MAX(epoch)*$1) AS slot FROM epochs - WHERE finalized + WHERE epoch <= $2 AND rewards_exported ) SELECT epoch @@ -187,7 +198,7 @@ func main() { AND slot < (SELECT slot FROM last_exported_epoch) GROUP BY epoch ORDER BY epoch; - `, utils.Config.Chain.ClConfig.SlotsPerEpoch) + `, utils.Config.Chain.ClConfig.SlotsPerEpoch, latestFinalizedEpoch) if err != nil { utils.LogError(err, "Error getting epochs with missing slot status from db", 0) return @@ -220,6 +231,94 @@ func main() { indexMissingBlocks(opts.StartBlock, opts.EndBlock, bt, erigonClient) case "migrate-last-attestation-slot-bigtable": migrateLastAttestationSlotToBigtable() + case "export-genesis-validators": + validators, err := rpcClient.GetValidatorState(0) + if err != nil { + logrus.Fatalf("error retrieving genesis validator state") + } + + validatorsArr := make([]*types.Validator, 0, len(validators.Data)) + + for _, validator := range validators.Data { + validatorsArr = append(validatorsArr, &types.Validator{ + Index: uint64(validator.Index), + PublicKey: utils.MustParseHex(validator.Validator.Pubkey), + WithdrawalCredentials: utils.MustParseHex(validator.Validator.WithdrawalCredentials), + Balance: uint64(validator.Balance), + EffectiveBalance: uint64(validator.Validator.EffectiveBalance), + Slashed: validator.Validator.Slashed, + ActivationEligibilityEpoch: uint64(validator.Validator.ActivationEligibilityEpoch), + ActivationEpoch: uint64(validator.Validator.ActivationEpoch), + ExitEpoch: uint64(validator.Validator.ExitEpoch), + WithdrawableEpoch: uint64(validator.Validator.WithdrawableEpoch), + Status: validator.Status, + }) + } + + batchSize := 10000 + for i := 0; i < len(validatorsArr); i += batchSize { + data := &types.EpochData{ + SyncDuties: make(map[types.Slot]map[types.ValidatorIndex]bool), + AttestationDuties: make(map[types.Slot]map[types.ValidatorIndex][]types.Slot), + ValidatorAssignmentes: &types.EpochAssignments{ + ProposerAssignments: map[uint64]uint64{}, + AttestorAssignments: map[string]uint64{}, + SyncAssignments: make([]uint64, 0), + }, + Blocks: make(map[uint64]map[string]*types.Block), + FutureBlocks: make(map[uint64]map[string]*types.Block), + EpochParticipationStats: &types.ValidatorParticipation{}, + Finalized: false, + } + + data.Validators = make([]*types.Validator, 0, batchSize) + + start := i + end := i + batchSize + if end >= len(validatorsArr) { + end = len(validatorsArr) - 1 + } + data.Validators = append(data.Validators, validatorsArr[start:end]...) + + logrus.Infof("saving validators %v-%v", data.Validators[0].Index, data.Validators[len(data.Validators)-1].Index) + tx := db.WriterDb.MustBegin() + + err = db.SaveValidators(data, tx, rpcClient, len(data.Validators)) + if err != nil { + logrus.Fatal(err) + } + err = tx.Commit() + if err != nil { + logrus.Fatal(err) + } + } + + _, err = db.WriterDb.Exec(` + INSERT INTO blocks_deposits (block_slot, block_index, publickey, withdrawalcredentials, amount, signature, valid_signature) + SELECT + 0 as block_slot, + v.validatorindex as block_index, + v.pubkey as publickey, + v.withdrawalcredentials, + 32*1e9 as amount, + '\x'::bytea as signature, + true + FROM validators v ON CONFLICT DO NOTHING`) + if err != nil { + logrus.Fatal(err) + } + _, err = db.WriterDb.Exec(` + INSERT INTO blocks (epoch, slot, blockroot, parentroot, stateroot, signature, syncaggregate_participation, proposerslashingscount, attesterslashingscount, attestationscount, depositscount, withdrawalcount, voluntaryexitscount, proposer, status, exec_transactions_count, eth1data_depositcount) + VALUES (0, 0, '\x'::bytea, '\x'::bytea, '\x'::bytea, '\x'::bytea, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0) + ON CONFLICT (slot, blockroot) DO NOTHING`) + if err != nil { + logrus.Fatal(err) + } + + err = db.BigtableClient.SaveValidatorBalances(0, validatorsArr) + if err != nil { + logrus.Fatal(err) + } default: utils.LogFatal(nil, fmt.Sprintf("unknown command %s", opts.Command), 0) } @@ -245,7 +344,7 @@ func DebugBlocks() error { } // logrus.WithFields(logrus.Fields{"block": i, "data": fmt.Sprintf("%+v", b)}).Infof("block from bt") - cb, _, err := client.GetBlock(int64(i)) + cb, _, err := client.GetBlock(int64(i), "parity/geth") if err != nil { return err } @@ -409,20 +508,42 @@ func updateAggreationBits(rpcClient *rpc.LighthouseClient, startEpoch uint64, en utils.LogError(err, fmt.Errorf("error getting Slot [%v] status", block.Slot), 0) return } + importWholeBlock := false if status != block.Status { logrus.Infof("Slot[%v] has the wrong status [%v], but should be [%v]", block.Slot, status, block.Status) if block.Status == 1 { - err := db.SaveBlock(block) - if err != nil { - utils.LogError(err, fmt.Errorf("error saving Slot [%v]", block.Slot), 0) - return - } - continue + importWholeBlock = true } else { utils.LogError(err, fmt.Errorf("error on Slot [%v] - no update process for status [%v]", block.Slot, block.Status), 0) return } + } else if len(block.Attestations) > 0 { + count := 0 + err := db.ReaderDb.Get(&count, ` + SELECT COUNT(*) + FROM + blocks_attestations + WHERE + block_slot=$1`, block.Slot) + if err != nil { + utils.LogError(err, fmt.Errorf("error getting Slot [%v] status", block.Slot), 0) + return + } + // We only know about cases where we have no attestations in the db but the node has one. + // So we don't handle cases (for now) where there are attestations with different sizes - that would require a different handling + if count == 0 { + importWholeBlock = true + } + } + + if importWholeBlock { + err := db.SaveBlock(block, true) + if err != nil { + utils.LogError(err, fmt.Errorf("error saving Slot [%v]", block.Slot), 0) + return + } + continue } for i, a := range block.Attestations { @@ -554,7 +675,7 @@ func CompareRewards(dayStart uint64, dayEnd uint64, validator uint64, bt *db.Big err = db.ReaderDb.Get(&dbRewards, ` SELECT COALESCE(cl_rewards_gwei, 0) AS cl_rewards_gwei - FROM validator_stats WHERE day = $1 and validatorindex = $2`, day, validator) + FROM validator_stats WHERE validatorindex = $2 AND day = $1`, day, validator) if err != nil { logrus.Fatalf("error getting cl_rewards_gwei from db: %v", err) return @@ -627,7 +748,7 @@ func indexMissingBlocks(start uint64, end uint64, bt *db.Bigtable, client *rpc.E logrus.Infof("block [%v] not found so we need to index it", j) if _, err := db.BigtableClient.GetBlockFromBlocksTable(j); err != nil { logrus.Infof("could not load [%v] from blocks table so we need to fetch it from the node and save it", j) - bc, _, err := client.GetBlock(int64(j)) + bc, _, err := client.GetBlock(int64(j), "parity/geth") if err != nil { utils.LogError(err, fmt.Sprintf("error getting block: %v from ethereum node", j), 0) } diff --git a/cmd/notification-collector/main.go b/cmd/notification-collector/main.go index 06dc05253e..9518e91d5b 100644 --- a/cmd/notification-collector/main.go +++ b/cmd/notification-collector/main.go @@ -105,6 +105,9 @@ func main() { db.BigtableClient = bt }() + if utils.Config.TieredCacheProvider != "redis" { + logrus.Fatalf("no cache provider set, please set TierdCacheProvider (redis)") + } if utils.Config.TieredCacheProvider == "redis" || len(utils.Config.RedisCacheEndpoint) != 0 { wg.Add(1) go func() { @@ -115,14 +118,6 @@ func main() { } wg.Wait() - if utils.Config.TieredCacheProvider == "bigtable" && len(utils.Config.RedisCacheEndpoint) == 0 { - cache.MustInitTieredCacheBigtable(db.BigtableClient.GetClient(), fmt.Sprintf("%d", utils.Config.Chain.ClConfig.DepositChainID)) - logrus.Infof("tiered Cache initialized, latest finalized epoch: %v", services.LatestFinalizedEpoch()) - } - - if utils.Config.TieredCacheProvider != "bigtable" && utils.Config.TieredCacheProvider != "redis" { - logrus.Fatalf("no cache provider set, please set TierdCacheProvider (example redis, bigtable)") - } defer db.ReaderDb.Close() defer db.WriterDb.Close() diff --git a/cmd/notification-sender/main.go b/cmd/notification-sender/main.go index 566ccb5500..dc6188a42e 100644 --- a/cmd/notification-sender/main.go +++ b/cmd/notification-sender/main.go @@ -105,6 +105,10 @@ func main() { db.BigtableClient = bt }() + if utils.Config.TieredCacheProvider != "redis" { + logrus.Fatalf("no cache provider set, please set TierdCacheProvider (redis)") + } + if utils.Config.TieredCacheProvider == "redis" || len(utils.Config.RedisCacheEndpoint) != 0 { wg.Add(1) go func() { @@ -115,14 +119,6 @@ func main() { } wg.Wait() - if utils.Config.TieredCacheProvider == "bigtable" && len(utils.Config.RedisCacheEndpoint) == 0 { - cache.MustInitTieredCacheBigtable(db.BigtableClient.GetClient(), fmt.Sprintf("%d", utils.Config.Chain.ClConfig.DepositChainID)) - logrus.Infof("tiered Cache initialized, latest finalized epoch: %v", services.LatestFinalizedEpoch()) - } - - if utils.Config.TieredCacheProvider != "bigtable" && utils.Config.TieredCacheProvider != "redis" { - logrus.Fatalf("no cache provider set, please set TierdCacheProvider (example redis, bigtable)") - } defer db.ReaderDb.Close() defer db.WriterDb.Close() diff --git a/cmd/rewards-exporter/main.go b/cmd/rewards-exporter/main.go index 6af53b1e3d..c76e1214ee 100644 --- a/cmd/rewards-exporter/main.go +++ b/cmd/rewards-exporter/main.go @@ -1,6 +1,7 @@ package main import ( + "eth2-exporter/cache" "eth2-exporter/db" "eth2-exporter/services" "eth2-exporter/types" @@ -65,6 +66,24 @@ func main() { defer db.ReaderDb.Close() defer db.WriterDb.Close() + if bnAddress == nil || *bnAddress == "" { + if utils.Config.Indexer.Node.Host == "" { + utils.LogFatal(nil, "no beacon node url provided", 0) + } else { + logrus.Info("applying becon node endpoint from config") + *bnAddress = fmt.Sprintf("http://%s:%s", utils.Config.Indexer.Node.Host, utils.Config.Indexer.Node.Port) + } + } + + if enAddress == nil || *enAddress == "" { + if utils.Config.Eth1ErigonEndpoint == "" { + utils.LogFatal(nil, "no execution node url provided", 0) + } else { + logrus.Info("applying execution node endpoint from config") + *enAddress = utils.Config.Eth1ErigonEndpoint + } + } + client := beacon.NewClient(*bnAddress, time.Minute*5) bt, err := db.InitBigtable(utils.Config.Bigtable.Project, utils.Config.Bigtable.Instance, fmt.Sprintf("%d", utils.Config.Chain.ClConfig.DepositChainID), utils.Config.RedisCacheEndpoint) @@ -73,14 +92,22 @@ func main() { } defer bt.Close() + cache.MustInitTieredCache(utils.Config.RedisCacheEndpoint) + logrus.Infof("tiered Cache initialized, latest finalized epoch: %v", services.LatestFinalizedEpoch()) + if *epochEnd != 0 { + latestFinalizedEpoch := services.LatestFinalizedEpoch() + if *epochEnd > latestFinalizedEpoch { + logrus.Errorf("error epochEnd [%v] is greater then latestFinalizedEpoch [%v]", epochEnd, latestFinalizedEpoch) + return + } g := errgroup.Group{} g.SetLimit(*batchConcurrency) start := time.Now() epochsCompleted := int64(0) notExportedEpochs := []uint64{} - err = db.WriterDb.Select(¬ExportedEpochs, "SELECT epoch FROM epochs WHERE finalized AND NOT rewards_exported AND epoch >= $1 AND epoch <= $2 ORDER BY epoch DESC", *epochStart, *epochEnd) + err = db.WriterDb.Select(¬ExportedEpochs, "SELECT epoch FROM epochs WHERE NOT rewards_exported AND epoch >= $1 AND epoch <= $2 ORDER BY epoch DESC", *epochStart, *epochEnd) if err != nil { logrus.Fatal(err) } @@ -142,8 +169,9 @@ func main() { if *epoch == -1 { lastExportedEpoch := uint64(0) for { + latestFinalizedEpoch := services.LatestFinalizedEpoch() notExportedEpochs := []uint64{} - err = db.WriterDb.Select(¬ExportedEpochs, "SELECT epoch FROM epochs WHERE finalized AND NOT rewards_exported AND epoch > $1 ORDER BY epoch desc LIMIT 10", lastExportedEpoch) + err = db.WriterDb.Select(¬ExportedEpochs, "SELECT epoch FROM epochs WHERE NOT rewards_exported AND epoch > $1 AND epoch <= $2 ORDER BY epoch desc LIMIT 10", lastExportedEpoch, latestFinalizedEpoch) if err != nil { utils.LogFatal(err, "getting chain head from lighthouse error", 0) } @@ -173,6 +201,11 @@ func main() { } } + latestFinalizedEpoch := services.LatestFinalizedEpoch() + if *epoch > int64(latestFinalizedEpoch) { + logrus.Errorf("error epoch [%v] is greater then latestFinalizedEpoch [%v]", epoch, latestFinalizedEpoch) + return + } err = export(uint64(*epoch), bt, client, enAddress) if err != nil { logrus.Fatal(err) diff --git a/cmd/signatures/main.go b/cmd/signatures/main.go index e92f07db7a..19e4f5bd73 100644 --- a/cmd/signatures/main.go +++ b/cmd/signatures/main.go @@ -10,7 +10,7 @@ import ( "eth2-exporter/version" "flag" "fmt" - "io/ioutil" + "io" "net/http" "time" @@ -202,7 +202,7 @@ func GetNextSignatures(bt *db.Bigtable, page string, status types.SignatureImpor return nil, nil, fmt.Errorf("error querying signatures api: %v", resp.Status) } - body, err := ioutil.ReadAll(resp.Body) + body, err := io.ReadAll(resp.Body) if err != nil { return nil, nil, err } diff --git a/cmd/statistics/main.go b/cmd/statistics/main.go index b7692f9f50..0b9c398dc9 100644 --- a/cmd/statistics/main.go +++ b/cmd/statistics/main.go @@ -62,6 +62,13 @@ func main() { } utils.Config = cfg + if utils.Config.Chain.ClConfig.SlotsPerEpoch == 0 || utils.Config.Chain.ClConfig.SecondsPerSlot == 0 { + utils.LogFatal(fmt.Errorf("error ether SlotsPerEpoch [%v] or SecondsPerSlot [%v] are not set", utils.Config.Chain.ClConfig.SlotsPerEpoch, utils.Config.Chain.ClConfig.SecondsPerSlot), "", 0) + return + } else { + logrus.Infof("Writing statistic with: SlotsPerEpoch [%v] or SecondsPerSlot [%v]", utils.Config.Chain.ClConfig.SlotsPerEpoch, utils.Config.Chain.ClConfig.SecondsPerSlot) + } + db.MustInitDB(&types.DatabaseConfig{ Username: cfg.WriterDatabase.Username, Password: cfg.WriterDatabase.Password, @@ -109,14 +116,12 @@ func main() { price.Init(utils.Config.Chain.ClConfig.DepositChainID, utils.Config.Eth1ErigonEndpoint) - if utils.Config.TieredCacheProvider == "redis" || len(utils.Config.RedisCacheEndpoint) != 0 { - cache.MustInitTieredCache(utils.Config.RedisCacheEndpoint) - } else if utils.Config.TieredCacheProvider == "bigtable" && len(utils.Config.RedisCacheEndpoint) == 0 { - cache.MustInitTieredCacheBigtable(db.BigtableClient.GetClient(), fmt.Sprintf("%d", utils.Config.Chain.ClConfig.DepositChainID)) + if utils.Config.TieredCacheProvider != "redis" { + logrus.Fatalf("No cache provider set. Please set TierdCacheProvider (example redis)") } - if utils.Config.TieredCacheProvider != "bigtable" && utils.Config.TieredCacheProvider != "redis" { - logrus.Fatalf("No cache provider set. Please set TierdCacheProvider (example redis, bigtable)") + if utils.Config.TieredCacheProvider == "redis" || len(utils.Config.RedisCacheEndpoint) != 0 { + cache.MustInitTieredCache(utils.Config.RedisCacheEndpoint) } if opt.statisticsDaysToExport != "" { @@ -141,7 +146,7 @@ func main() { err = db.WriteValidatorStatisticsForDay(uint64(d), opt.concurrencyTotal, opt.concurrencyCl, opt.concurrencyFailedAttestations) if err != nil { - logrus.Errorf("error exporting stats for day %v: %v", d, err) + utils.LogError(err, fmt.Errorf("error exporting stats for day %v", d), 0) break } } @@ -181,7 +186,7 @@ func main() { err = db.WriteValidatorStatisticsForDay(uint64(opt.statisticsDayToExport), opt.concurrencyTotal, opt.concurrencyCl, opt.concurrencyFailedAttestations) if err != nil { - logrus.Errorf("error exporting stats for day %v: %v", opt.statisticsDayToExport, err) + utils.LogError(err, fmt.Errorf("error exporting stats for day %v", opt.statisticsDayToExport), 0) } } @@ -232,6 +237,7 @@ func statisticsLoop(concurrencyTotal uint64, concurrencyCl uint64, concurrencyFa currentDay := latestEpoch / epochsPerDay previousDay := currentDay - 1 + logrus.Infof("Performing statisticsLoop with currentDay %v and previousDay %v", currentDay, previousDay) if previousDay > currentDay { previousDay = currentDay } @@ -250,7 +256,7 @@ func statisticsLoop(concurrencyTotal uint64, concurrencyCl uint64, concurrencyFa for day := lastExportedDayValidator; day <= previousDay; day++ { err := db.WriteValidatorStatisticsForDay(day, concurrencyTotal, concurrencyCl, concurrencyFailedAttestations) if err != nil { - logrus.Errorf("error exporting stats for day %v: %v", day, err) + utils.LogError(err, fmt.Errorf("error exporting stats for day %v", day), 0) break } } diff --git a/config/config.go b/config/config.go index 07daa19618..5512664bc3 100644 --- a/config/config.go +++ b/config/config.go @@ -2,32 +2,26 @@ package config import _ "embed" -//go:embed mainnet.preset.yml -var MainnetPresetYml string // https://github.com/ethereum/consensus-specs - -//go:embed minimal.preset.yml -var MinimalPresetYml string // https://github.com/ethereum/consensus-specs - -//go:embed gnosis.preset.yml -var GnosisPresetYml string // https://github.com/gnosischain/configs - -//go:embed mainnet.chain.yml -var MainnetChainYml string // https://github.com/ethereum/consensus-specs +//go:embed default.config.yml +var DefaultConfigYml string //go:embed mainnet.chain.yml -var MinimalChainYml string // https://github.com/ethereum/consensus-specs +var MainnetChainYml string //go:embed prater.chain.yml -var PraterChainYml string // https://github.com/eth-clients/goerli/blob/d6a227e/prater/config.yaml +var PraterChainYml string + +//go:embed ropsten.chain.yml +var RopstenChainYml string //go:embed sepolia.chain.yml -var SepoliaChainYml string // https://github.com/eth-clients/sepolia/blob/main/bepolia/config.yaml +var SepoliaChainYml string -//go:embed gnosis.chain.yml -var GnosisChainYml string // https://github.com/gnosischain/configs/blob/main/mainnet/config.yaml +//go:embed testnet.chain.yml +var TestnetChainYml string -//go:embed dencun-devnet-8.chain.yml -var DencunDevnet8ChainYml string // https://github.com/ethpandaops/dencun-testnet/blob/83e4547/network-configs/devnet-8/config.yaml +//go:embed gnosis.chain.yml +var GnosisChainYml string -//go:embed default.config.yml -var DefaultConfigYml string +//go:embed holesky.chain.yml +var HoleskyChainYml string diff --git a/config/gnosis.chain.yml b/config/gnosis.chain.yml index fa2bac7195..6f4ba71e5c 100644 --- a/config/gnosis.chain.yml +++ b/config/gnosis.chain.yml @@ -1,13 +1,31 @@ -CONFIG_NAME: gnosis -PRESET_BASE: gnosis +# Gnosis config -# Transition +# Extends the mainnet preset +PRESET_BASE: 'mainnet' + +# Free-form short name of the network that this configuration applies to - known +# canonical network names include: +# * 'mainnet' - there can be only one +# * 'prater' - testnet +# Must match the regex: [a-z0-9\-] +CONFIG_NAME: 'gnosis' + + +# Forking # --------------------------------------------------------------- -# Estimated on Dec 5, 2022 -TERMINAL_TOTAL_DIFFICULTY: 8626000000000000000000058750000000000000000000 -# By default, don't use these params -TERMINAL_BLOCK_HASH: 0x0000000000000000000000000000000000000000000000000000000000000000 -TERMINAL_BLOCK_HASH_ACTIVATION_EPOCH: 18446744073709551615 +# Some forks are disabled for now: +# - These may be re-assigned to another fork-version later +# - Temporarily set to max uint64 value: 2**64 - 1 + +# Altair +ALTAIR_FORK_VERSION: 0x01000064 +ALTAIR_FORK_EPOCH: 512 +# Bellatrix +BELLATRIX_FORK_VERSION: 0x02000064 +BELLATRIX_FORK_EPOCH: 18446744073709551615 +# Capella +CAPELLA_FORK_EPOCH: 18446744073709551615 + # Misc # --------------------------------------------------------------- @@ -33,6 +51,10 @@ HYSTERESIS_QUOTIENT: 4 HYSTERESIS_DOWNWARD_MULTIPLIER: 1 # 5 (plus 1.25) HYSTERESIS_UPWARD_MULTIPLIER: 5 +# Fork Choice +# --------------------------------------------------------------- +# 2**3 (= 8) +SAFE_SLOTS_TO_UPDATE_JUSTIFIED: 8 # Validator # --------------------------------------------------------------- # 2**10 (= 1024) ~1.4 hour @@ -129,6 +151,8 @@ MAX_ATTESTATIONS: 128 MAX_DEPOSITS: 16 # 2**4 (= 16) MAX_VOLUNTARY_EXITS: 16 +# 2**4 (= 16) +MAX_BLS_TO_EXECUTION_CHANGES: 16 # Signature domains # --------------------------------------------------------------- DOMAIN_BEACON_PROPOSER: 0x00000000 @@ -142,16 +166,27 @@ DOMAIN_SYNC_COMMITTEE: 0x07000000 DOMAIN_SYNC_COMMITTEE_SELECTION_PROOF: 0x08000000 DOMAIN_CONTRIBUTION_AND_PROOF: 0x09000000 -# Altair -ALTAIR_FORK_VERSION: 0x01000064 -ALTAIR_FORK_EPOCH: 512 -# Bellatrix -BELLATRIX_FORK_VERSION: 0x02000064 -BELLATRIX_FORK_EPOCH: 385536 # 2022-11-30T19:23:40.000Z -# Capella -CAPELLA_FORK_VERSION: 0x03000064 -CAPELLA_FORK_EPOCH: 648704 # 2023-08-01T11:34:20.000Z - INACTIVITY_SCORE_BIAS: 4 # 2**4 (= 16) -INACTIVITY_SCORE_RECOVERY_RATE: 16 \ No newline at end of file +INACTIVITY_SCORE_RECOVERY_RATE: 16 +INACTIVITY_PENALTY_QUOTIENT_ALTAIR: 50331648 +MIN_SLASHING_PENALTY_QUOTIENT_ALTAIR: 64 +PROPORTIONAL_SLASHING_MULTIPLIER_ALTAIR: 2 + +# Sync committee +# --------------------------------------------------------------- +# 2**9 (= 512) +SYNC_COMMITTEE_SIZE: 512 +# 2**9 (= 512) +# assert EPOCHS_PER_SYNC_COMMITTEE_PERIOD * SLOTS_PER_EPOCH <= SLOTS_PER_HISTORICAL_ROOT +EPOCHS_PER_SYNC_COMMITTEE_PERIOD: 512 + +# Execution +# --------------------------------------------------------------- +# 2**4 (= 16) withdrawals +MAX_WITHDRAWALS_PER_PAYLOAD: 16 + +# Withdrawals processing +# --------------------------------------------------------------- +# 2**14 (= 16384) validators +MAX_VALIDATORS_PER_WITHDRAWALS_SWEEP: 16384 \ No newline at end of file diff --git a/config/holesky.chain.yml b/config/holesky.chain.yml new file mode 100644 index 0000000000..054729a9f7 --- /dev/null +++ b/config/holesky.chain.yml @@ -0,0 +1,147 @@ +# Holesky config + +# Extends the mainnet preset +PRESET_BASE: "mainnet" + +# Free-form short name of the network that this configuration applies to - known +# canonical network names include: +# * 'mainnet' - there can be only one +# * 'prater' - testnet +# Must match the regex: [a-z0-9\-] +CONFIG_NAME: "holesky" + +# Genesis +# --------------------------------------------------------------- +# `2**14` (= 16,384) +MIN_GENESIS_ACTIVE_VALIDATOR_COUNT: 16384 +# Sep-15-2023 13:55:00 +UTC +MIN_GENESIS_TIME: 1694786100 +# Mainnet initial fork version, recommend altering for testnets +GENESIS_FORK_VERSION: 0x00017000 +# Genesis delay 5 mins +GENESIS_DELAY: 300 + +# Forking +# --------------------------------------------------------------- +# Some forks are disabled for now: +# - These may be re-assigned to another fork-version later +# - Temporarily set to max uint64 value: 2**64 - 1 + +# Altair +ALTAIR_FORK_VERSION: 0x10017000 +ALTAIR_FORK_EPOCH: 0 +# Merge +BELLATRIX_FORK_VERSION: 0x20017000 +BELLATRIX_FORK_EPOCH: 0 +TERMINAL_TOTAL_DIFFICULTY: 0 +TERMINAL_BLOCK_HASH: 0x0000000000000000000000000000000000000000000000000000000000000000 +TERMINAL_BLOCK_HASH_ACTIVATION_EPOCH: 18446744073709551615 +# Capella +CAPELLA_FORK_VERSION: 0x30017000 +CAPELLA_FORK_EPOCH: 256 +# Deneb +DENEB_FORK_VERSION: 0x40017000 +DENEB_FORK_EPOCH: 18446744073709551615 + +# Time parameters +# --------------------------------------------------------------- +# 12 seconds +SECONDS_PER_SLOT: 12 +# 14 (estimate from Eth1 mainnet) +SECONDS_PER_ETH1_BLOCK: 14 +# 2**8 (= 256) epochs ~27 hours +MIN_VALIDATOR_WITHDRAWABILITY_DELAY: 256 +# 2**8 (= 256) epochs ~27 hours +SHARD_COMMITTEE_PERIOD: 256 +# 2**11 (= 2,048) Eth1 blocks ~8 hours +ETH1_FOLLOW_DISTANCE: 2048 + +# Validator cycle +# --------------------------------------------------------------- +# 2**2 (= 4) +INACTIVITY_SCORE_BIAS: 4 +# 2**4 (= 16) +INACTIVITY_SCORE_RECOVERY_RATE: 16 +# 28,000,000,000 Gwei to ensure quicker ejection +EJECTION_BALANCE: 28000000000 +# 2**2 (= 4) +MIN_PER_EPOCH_CHURN_LIMIT: 4 +# 2**16 (= 65,536) +CHURN_LIMIT_QUOTIENT: 65536 + +# Fork choice +# --------------------------------------------------------------- +# 40% +PROPOSER_SCORE_BOOST: 40 + +# Deposit contract +# --------------------------------------------------------------- +# Ethereum +DEPOSIT_CHAIN_ID: 17000 +DEPOSIT_NETWORK_ID: 17000 +DEPOSIT_CONTRACT_ADDRESS: 0x4242424242424242424242424242424242424242 + +# Sync committee +# --------------------------------------------------------------- +# 2**9 (= 512) +SYNC_COMMITTEE_SIZE: 512 +# 2**8 (= 256) +EPOCHS_PER_SYNC_COMMITTEE_PERIOD: 256 + +# Networking +# --------------------------------------------------------------- +# `10 * 2**20` (= 10485760, 10 MiB) +GOSSIP_MAX_SIZE: 10485760 +# `2**10` (= 1024) +MAX_REQUEST_BLOCKS: 1024 +# `2**8` (= 256) +EPOCHS_PER_SUBNET_SUBSCRIPTION: 256 +# `MIN_VALIDATOR_WITHDRAWABILITY_DELAY + CHURN_LIMIT_QUOTIENT // 2` (= 33024, ~5 months) +MIN_EPOCHS_FOR_BLOCK_REQUESTS: 33024 +# `10 * 2**20` (=10485760, 10 MiB) +MAX_CHUNK_SIZE: 10485760 +# 5s +TTFB_TIMEOUT: 5 +# 10s +RESP_TIMEOUT: 10 +ATTESTATION_PROPAGATION_SLOT_RANGE: 32 +# 500ms +MAXIMUM_GOSSIP_CLOCK_DISPARITY: 500 +MESSAGE_DOMAIN_INVALID_SNAPPY: 0x00000000 +MESSAGE_DOMAIN_VALID_SNAPPY: 0x01000000 +# 2 subnets per node +SUBNETS_PER_NODE: 2 +# 2**8 (= 64) +ATTESTATION_SUBNET_COUNT: 64 +ATTESTATION_SUBNET_EXTRA_BITS: 0 +# ceillog2(ATTESTATION_SUBNET_COUNT) + ATTESTATION_SUBNET_EXTRA_BITS +ATTESTATION_SUBNET_PREFIX_BITS: 6 + +# Deneb +# `2**7` (=128) +MAX_REQUEST_BLOCKS_DENEB: 128 +# MAX_REQUEST_BLOCKS_DENEB * MAX_BLOBS_PER_BLOCK +MAX_REQUEST_BLOB_SIDECARS: 768 +# `2**12` (= 4096 epochs, ~18 days) +MIN_EPOCHS_FOR_BLOB_SIDECARS_REQUESTS: 4096 +# `6` +BLOB_SIDECAR_SUBNET_COUNT: 6 +# `uint64(6)` +MAX_BLOBS_PER_BLOCK: 6 + +# Time parameters +# --------------------------------------------------------------- +# 2**0 (= 1) slots 12 seconds +MIN_ATTESTATION_INCLUSION_DELAY: 1 +# 2**5 (= 32) slots 6.4 minutes +SLOTS_PER_EPOCH: 32 +# 2**0 (= 1) epochs 6.4 minutes +MIN_SEED_LOOKAHEAD: 1 +# 2**2 (= 4) epochs 25.6 minutes +MAX_SEED_LOOKAHEAD: 4 +# 2**6 (= 64) epochs ~6.8 hours +EPOCHS_PER_ETH1_VOTING_PERIOD: 64 +# 2**13 (= 8,192) slots ~27 hours +SLOTS_PER_HISTORICAL_ROOT: 8192 +# 2**2 (= 4) epochs 25.6 minutes +MIN_EPOCHS_TO_INACTIVITY_PENALTY: 4 \ No newline at end of file diff --git a/config/mainnet.chain.yml b/config/mainnet.chain.yml index 36b31c43c6..b2fd6b36a1 100644 --- a/config/mainnet.chain.yml +++ b/config/mainnet.chain.yml @@ -1,27 +1,23 @@ -# content of this file is https://github.com/ethereum/consensus-specs/blob/59fd2a6/configs/mainnet.yaml - # Mainnet config # Extends the mainnet preset -PRESET_BASE: 'mainnet' +PRESET_BASE: "mainnet" # Free-form short name of the network that this configuration applies to - known # canonical network names include: # * 'mainnet' - there can be only one # * 'prater' - testnet # Must match the regex: [a-z0-9\-] -CONFIG_NAME: 'mainnet' +CONFIG_NAME: "mainnet" # Transition # --------------------------------------------------------------- -# Estimated on Sept 15, 2022 -TERMINAL_TOTAL_DIFFICULTY: 58750000000000000000000 +# TBD, 2**256-2**10 is a placeholder +TERMINAL_TOTAL_DIFFICULTY: 115792089237316195423570985008687907853269984665640564039457584007913129638912 # By default, don't use these params TERMINAL_BLOCK_HASH: 0x0000000000000000000000000000000000000000000000000000000000000000 TERMINAL_BLOCK_HASH_ACTIVATION_EPOCH: 18446744073709551615 - - # Genesis # --------------------------------------------------------------- # `2**14` (= 16,384) @@ -33,7 +29,6 @@ GENESIS_FORK_VERSION: 0x00000000 # 604800 seconds (7 days) GENESIS_DELAY: 604800 - # Forking # --------------------------------------------------------------- # Some forks are disabled for now: @@ -42,26 +37,20 @@ GENESIS_DELAY: 604800 # Altair ALTAIR_FORK_VERSION: 0x01000000 -ALTAIR_FORK_EPOCH: 74240 # Oct 27, 2021, 10:56:23am UTC +ALTAIR_FORK_EPOCH: 74240 # Oct 27, 2021, 10:56:23am UTC # Bellatrix BELLATRIX_FORK_VERSION: 0x02000000 -BELLATRIX_FORK_EPOCH: 144896 # Sept 6, 2022, 11:34:47am UTC +BELLATRIX_FORK_EPOCH: 144896 # Sept 6, 2022, 11:34:47am UTC # Capella CAPELLA_FORK_VERSION: 0x03000000 -CAPELLA_FORK_EPOCH: 194048 # April 12, 2023, 10:27:35pm UTC +CAPELLA_FORK_EPOCH: 194048 # April 12, 2023, 10:27:35pm UTC # Deneb DENEB_FORK_VERSION: 0x04000000 DENEB_FORK_EPOCH: 18446744073709551615 -# EIP6110 -EIP6110_FORK_VERSION: 0x05000000 # temporary stub -EIP6110_FORK_EPOCH: 18446744073709551615 -# EIP7002 -EIP7002_FORK_VERSION: 0x05000000 # temporary stub -EIP7002_FORK_EPOCH: 18446744073709551615 -# WHISK -WHISK_FORK_VERSION: 0x06000000 # temporary stub -WHISK_FORK_EPOCH: 18446744073709551615 - +# Byzantium +BYZANTIUM_FORK_BLOCK: 4370000 +# Constantinople +CONSTANTINOPLE_FORK_BLOCK: 7280000 # Time parameters # --------------------------------------------------------------- @@ -76,7 +65,6 @@ SHARD_COMMITTEE_PERIOD: 256 # 2**11 (= 2,048) Eth1 blocks ~8 hours ETH1_FOLLOW_DISTANCE: 2048 - # Validator cycle # --------------------------------------------------------------- # 2**2 (= 4) @@ -90,7 +78,6 @@ MIN_PER_EPOCH_CHURN_LIMIT: 4 # 2**16 (= 65,536) CHURN_LIMIT_QUOTIENT: 65536 - # Fork choice # --------------------------------------------------------------- # 40% @@ -103,42 +90,217 @@ DEPOSIT_CHAIN_ID: 1 DEPOSIT_NETWORK_ID: 1 DEPOSIT_CONTRACT_ADDRESS: 0x00000000219ab540356cBB839Cbe05303d7705Fa +# Mainnet preset - Altair -# Networking -# --------------------------------------------------------------- -# `10 * 2**20` (= 10485760, 10 MiB) -GOSSIP_MAX_SIZE: 10485760 -# `2**10` (= 1024) -MAX_REQUEST_BLOCKS: 1024 -# `2**8` (= 256) -EPOCHS_PER_SUBNET_SUBSCRIPTION: 256 -# `MIN_VALIDATOR_WITHDRAWABILITY_DELAY + CHURN_LIMIT_QUOTIENT // 2` (= 33024, ~5 months) -MIN_EPOCHS_FOR_BLOCK_REQUESTS: 33024 -# `10 * 2**20` (=10485760, 10 MiB) -MAX_CHUNK_SIZE: 10485760 -# 5s -TTFB_TIMEOUT: 5 -# 10s -RESP_TIMEOUT: 10 -ATTESTATION_PROPAGATION_SLOT_RANGE: 32 -# 500ms -MAXIMUM_GOSSIP_CLOCK_DISPARITY: 500 -MESSAGE_DOMAIN_INVALID_SNAPPY: 0x00000000 -MESSAGE_DOMAIN_VALID_SNAPPY: 0x01000000 -# 2 subnets per node -SUBNETS_PER_NODE: 2 -# 2**8 (= 64) -ATTESTATION_SUBNET_COUNT: 64 -ATTESTATION_SUBNET_EXTRA_BITS: 0 -# ceillog2(ATTESTATION_SUBNET_COUNT) + ATTESTATION_SUBNET_EXTRA_BITS -ATTESTATION_SUBNET_PREFIX_BITS: 6 +# Updated penalty values +# --------------------------------------------------------------- +# 3 * 2**24 (= 50,331,648) +INACTIVITY_PENALTY_QUOTIENT_ALTAIR: 50331648 +# 2**6 (= 64) +MIN_SLASHING_PENALTY_QUOTIENT_ALTAIR: 64 +# 2 +PROPORTIONAL_SLASHING_MULTIPLIER_ALTAIR: 2 -# Deneb -# `2**7` (=128) -MAX_REQUEST_BLOCKS_DENEB: 128 -# MAX_REQUEST_BLOCKS_DENEB * MAX_BLOBS_PER_BLOCK -MAX_REQUEST_BLOB_SIDECARS: 768 -# `2**12` (= 4096 epochs, ~18 days) -MIN_EPOCHS_FOR_BLOB_SIDECARS_REQUESTS: 4096 -# `6` -BLOB_SIDECAR_SUBNET_COUNT: 6 \ No newline at end of file +# Sync committee +# --------------------------------------------------------------- +# 2**9 (= 512) +SYNC_COMMITTEE_SIZE: 512 +# 2**8 (= 256) +EPOCHS_PER_SYNC_COMMITTEE_PERIOD: 256 + +# Sync protocol +# --------------------------------------------------------------- +# 1 +MIN_SYNC_COMMITTEE_PARTICIPANTS: 1 +# SLOTS_PER_EPOCH * EPOCHS_PER_SYNC_COMMITTEE_PERIOD (= 32 * 256) +UPDATE_TIMEOUT: 8192 +# Mainnet preset - Bellatrix + +# Updated penalty values +# --------------------------------------------------------------- +# 2**24 (= 16,777,216) +INACTIVITY_PENALTY_QUOTIENT_BELLATRIX: 16777216 +# 2**5 (= 32) +MIN_SLASHING_PENALTY_QUOTIENT_BELLATRIX: 32 +# 3 +PROPORTIONAL_SLASHING_MULTIPLIER_BELLATRIX: 3 + +# Execution +# --------------------------------------------------------------- +# 2**30 (= 1,073,741,824) +MAX_BYTES_PER_TRANSACTION: 1073741824 +# 2**20 (= 1,048,576) +MAX_TRANSACTIONS_PER_PAYLOAD: 1048576 +# 2**8 (= 256) +BYTES_PER_LOGS_BLOOM: 256 +# 2**5 (= 32) +MAX_EXTRA_DATA_BYTES: 32 +# Minimal preset - Capella +# Mainnet preset - Custody Game + +# Time parameters +# --------------------------------------------------------------- +# 2**1 (= 2) epochs, 12.8 minutes +RANDAO_PENALTY_EPOCHS: 2 +# 2**15 (= 32,768) epochs, ~146 days +EARLY_DERIVED_SECRET_PENALTY_MAX_FUTURE_EPOCHS: 32768 +# 2**14 (= 16,384) epochs ~73 days +EPOCHS_PER_CUSTODY_PERIOD: 16384 +# 2**11 (= 2,048) epochs, ~9 days +CUSTODY_PERIOD_TO_RANDAO_PADDING: 2048 +# 2**15 (= 32,768) epochs, ~146 days +MAX_CHUNK_CHALLENGE_DELAY: 32768 + +# Max operations +# --------------------------------------------------------------- +# 2**8 (= 256) +MAX_CUSTODY_KEY_REVEALS: 256 +# 2**0 (= 1) +MAX_EARLY_DERIVED_SECRET_REVEALS: 1 +# 2**2 (= 2) +MAX_CUSTODY_CHUNK_CHALLENGES: 4 +# 2** 4 (= 16) +MAX_CUSTODY_CHUNK_CHALLENGE_RESP: 16 +# 2**0 (= 1) +MAX_CUSTODY_SLASHINGS: 1 + +# Reward and penalty quotients +# --------------------------------------------------------------- +EARLY_DERIVED_SECRET_REVEAL_SLOT_REWARD_MULTIPLE: 2 +# 2**8 (= 256) +MINOR_REWARD_QUOTIENT: 256 +# Mainnet preset - Phase0 + +# Misc +# --------------------------------------------------------------- +# 2**6 (= 64) +MAX_COMMITTEES_PER_SLOT: 64 +# 2**7 (= 128) +TARGET_COMMITTEE_SIZE: 128 +# 2**11 (= 2,048) +MAX_VALIDATORS_PER_COMMITTEE: 2048 +# See issue 563 +SHUFFLE_ROUND_COUNT: 90 +# 4 +HYSTERESIS_QUOTIENT: 4 +# 1 (minus 0.25) +HYSTERESIS_DOWNWARD_MULTIPLIER: 1 +# 5 (plus 1.25) +HYSTERESIS_UPWARD_MULTIPLIER: 5 + +# Fork Choice +# --------------------------------------------------------------- +# 2**3 (= 8) +SAFE_SLOTS_TO_UPDATE_JUSTIFIED: 8 + +# Gwei values +# --------------------------------------------------------------- +# 2**0 * 10**9 (= 1,000,000,000) Gwei +MIN_DEPOSIT_AMOUNT: 1000000000 +# 2**5 * 10**9 (= 32,000,000,000) Gwei +MAX_EFFECTIVE_BALANCE: 32000000000 +# 2**0 * 10**9 (= 1,000,000,000) Gwei +EFFECTIVE_BALANCE_INCREMENT: 1000000000 + +# Time parameters +# --------------------------------------------------------------- +# 2**0 (= 1) slots 12 seconds +MIN_ATTESTATION_INCLUSION_DELAY: 1 +# 2**5 (= 32) slots 6.4 minutes +SLOTS_PER_EPOCH: 32 +# 2**0 (= 1) epochs 6.4 minutes +MIN_SEED_LOOKAHEAD: 1 +# 2**2 (= 4) epochs 25.6 minutes +MAX_SEED_LOOKAHEAD: 4 +# 2**6 (= 64) epochs ~6.8 hours +EPOCHS_PER_ETH1_VOTING_PERIOD: 64 +# 2**13 (= 8,192) slots ~27 hours +SLOTS_PER_HISTORICAL_ROOT: 8192 +# 2**2 (= 4) epochs 25.6 minutes +MIN_EPOCHS_TO_INACTIVITY_PENALTY: 4 + +# State list lengths +# --------------------------------------------------------------- +# 2**16 (= 65,536) epochs ~0.8 years +EPOCHS_PER_HISTORICAL_VECTOR: 65536 +# 2**13 (= 8,192) epochs ~36 days +EPOCHS_PER_SLASHINGS_VECTOR: 8192 +# 2**24 (= 16,777,216) historical roots, ~26,131 years +HISTORICAL_ROOTS_LIMIT: 16777216 +# 2**40 (= 1,099,511,627,776) validator spots +VALIDATOR_REGISTRY_LIMIT: 1099511627776 + +# Reward and penalty quotients +# --------------------------------------------------------------- +# 2**6 (= 64) +BASE_REWARD_FACTOR: 64 +# 2**9 (= 512) +WHISTLEBLOWER_REWARD_QUOTIENT: 512 +# 2**3 (= 8) +PROPOSER_REWARD_QUOTIENT: 8 +# 2**26 (= 67,108,864) +INACTIVITY_PENALTY_QUOTIENT: 67108864 +# 2**7 (= 128) (lower safety margin at Phase 0 genesis) +MIN_SLASHING_PENALTY_QUOTIENT: 128 +# 1 (lower safety margin at Phase 0 genesis) +PROPORTIONAL_SLASHING_MULTIPLIER: 1 + +# Max operations per block +# --------------------------------------------------------------- +# 2**4 (= 16) +MAX_PROPOSER_SLASHINGS: 16 +# 2**1 (= 2) +MAX_ATTESTER_SLASHINGS: 2 +# 2**7 (= 128) +MAX_ATTESTATIONS: 128 +# 2**4 (= 16) +MAX_DEPOSITS: 16 +# 2**4 (= 16) +MAX_VOLUNTARY_EXITS: 16 +# Mainnet preset - Sharding + +# Misc +# --------------------------------------------------------------- +# 2**10 (= 1,024) +MAX_SHARDS: 1024 +# 2**6 (= 64) +INITIAL_ACTIVE_SHARDS: 64 +# 2**3 (= 8) +SAMPLE_PRICE_ADJUSTMENT_COEFFICIENT: 8 +# 2**4 (= 16) +MAX_SHARD_PROPOSER_SLASHINGS: 16 +# +MAX_SHARD_HEADERS_PER_SHARD: 4 +# 2**8 (= 256) +SHARD_STATE_MEMORY_SLOTS: 256 +# 2**40 (= 1,099,511,627,776) +BLOB_BUILDER_REGISTRY_LIMIT: 1099511627776 + +# Shard blob samples +# --------------------------------------------------------------- +# 2**11 (= 2,048) +MAX_SAMPLES_PER_BLOCK: 2048 +# 2**10 (= 1,1024) +TARGET_SAMPLES_PER_BLOCK: 1024 + +# Gwei values +# --------------------------------------------------------------- +# 2**33 (= 8,589,934,592) Gwei +MAX_SAMPLE_PRICE: 8589934592 +# 2**3 (= 8) Gwei +MIN_SAMPLE_PRICE: 8 + +# Max operations per block +# --------------------------------------------------------------- +# 2**4 (= 16) +MAX_BLS_TO_EXECUTION_CHANGES: 16 + +# Execution +# --------------------------------------------------------------- +# [customized] 2**2 (= 4) +MAX_WITHDRAWALS_PER_PAYLOAD: 16 + +# Withdrawals processing +# --------------------------------------------------------------- +# [customized] 2**4 (= 16) validators +MAX_VALIDATORS_PER_WITHDRAWALS_SWEEP: 16384 diff --git a/config/minimal.chain.yml b/config/minimal.chain.yml index bdda4ec45a..53be612281 100644 --- a/config/minimal.chain.yml +++ b/config/minimal.chain.yml @@ -1,5 +1,3 @@ -# content of this file is https://github.com/ethereum/consensus-specs/blob/59fd2a6/configs/minimal.yaml - # Minimal config # Extends the minimal preset @@ -14,7 +12,7 @@ CONFIG_NAME: 'minimal' # Transition # --------------------------------------------------------------- -# 2**256-2**10 for testing minimal network +# TBD, 2**256-2**10 is a placeholder TERMINAL_TOTAL_DIFFICULTY: 115792089237316195423570985008687907853269984665640564039457584007913129638912 # By default, don't use these params TERMINAL_BLOCK_HASH: 0x0000000000000000000000000000000000000000000000000000000000000000 @@ -48,18 +46,9 @@ BELLATRIX_FORK_EPOCH: 18446744073709551615 # Capella CAPELLA_FORK_VERSION: 0x03000001 CAPELLA_FORK_EPOCH: 18446744073709551615 -# DENEB -DENEB_FORK_VERSION: 0x04000001 -DENEB_FORK_EPOCH: 18446744073709551615 -# EIP6110 -EIP6110_FORK_VERSION: 0x05000001 -EIP6110_FORK_EPOCH: 18446744073709551615 -# EIP7002 -EIP7002_FORK_VERSION: 0x05000001 -EIP7002_FORK_EPOCH: 18446744073709551615 -# WHISK -WHISK_FORK_VERSION: 0x06000001 -WHISK_FORK_EPOCH: 18446744073709551615 +# Sharding +SHARDING_FORK_VERSION: 0x04000001 +SHARDING_FORK_EPOCH: 18446744073709551615 # Time parameters @@ -104,42 +93,213 @@ DEPOSIT_NETWORK_ID: 5 # Configured on a per testnet basis DEPOSIT_CONTRACT_ADDRESS: 0x1234567890123456789012345678901234567890 +# Minimal preset - Altair + +# Updated penalty values +# --------------------------------------------------------------- +# 3 * 2**24 (= 50,331,648) +INACTIVITY_PENALTY_QUOTIENT_ALTAIR: 50331648 +# 2**6 (= 64) +MIN_SLASHING_PENALTY_QUOTIENT_ALTAIR: 64 +# 2 +PROPORTIONAL_SLASHING_MULTIPLIER_ALTAIR: 2 + + +# Sync committee +# --------------------------------------------------------------- +# [customized] +SYNC_COMMITTEE_SIZE: 32 +# [customized] +EPOCHS_PER_SYNC_COMMITTEE_PERIOD: 8 + + +# Sync protocol +# --------------------------------------------------------------- +# 1 +MIN_SYNC_COMMITTEE_PARTICIPANTS: 1 +# SLOTS_PER_EPOCH * EPOCHS_PER_SYNC_COMMITTEE_PERIOD (= 8 * 8) +UPDATE_TIMEOUT: 64 +# Minimal preset - Bellatrix + +# Updated penalty values +# --------------------------------------------------------------- +# 2**24 (= 16,777,216) +INACTIVITY_PENALTY_QUOTIENT_BELLATRIX: 16777216 +# 2**5 (= 32) +MIN_SLASHING_PENALTY_QUOTIENT_BELLATRIX: 32 +# 3 +PROPORTIONAL_SLASHING_MULTIPLIER_BELLATRIX: 3 + +# Execution +# --------------------------------------------------------------- +# 2**30 (= 1,073,741,824) +MAX_BYTES_PER_TRANSACTION: 1073741824 +# 2**20 (= 1,048,576) +MAX_TRANSACTIONS_PER_PAYLOAD: 1048576 +# 2**8 (= 256) +BYTES_PER_LOGS_BLOOM: 256 +# 2**5 (= 32) +MAX_EXTRA_DATA_BYTES: 32 +# Minimal preset - Capella +# Minimal preset - Custody Game + +# Time parameters +# --------------------------------------------------------------- +# 2**1 (= 2) epochs, 12.8 minutes +RANDAO_PENALTY_EPOCHS: 2 +# [customized] quicker for testing +EARLY_DERIVED_SECRET_PENALTY_MAX_FUTURE_EPOCHS: 64 +# [customized] quicker for testing +EPOCHS_PER_CUSTODY_PERIOD: 32 +# [customized] quicker for testing +CUSTODY_PERIOD_TO_RANDAO_PADDING: 8 +# [customize for faster testing] +MAX_CHUNK_CHALLENGE_DELAY: 64 + + +# Max operations +# --------------------------------------------------------------- +# 2**8 (= 256) +MAX_CUSTODY_KEY_REVEALS: 256 +# 2**0 (= 1) +MAX_EARLY_DERIVED_SECRET_REVEALS: 1 +# [customized] +MAX_CUSTODY_CHUNK_CHALLENGES: 2 +# [customized] +MAX_CUSTODY_CHUNK_CHALLENGE_RESP: 8 +# 2**0 (= 1) +MAX_CUSTODY_SLASHINGS: 1 + + +# Reward and penalty quotients +# --------------------------------------------------------------- +EARLY_DERIVED_SECRET_REVEAL_SLOT_REWARD_MULTIPLE: 2 +# 2**8 (= 256) +MINOR_REWARD_QUOTIENT: 256 +# Minimal preset - Phase0 + +# Misc +# --------------------------------------------------------------- +# [customized] Just 4 committees for slot for testing purposes +MAX_COMMITTEES_PER_SLOT: 4 +# [customized] unsecure, but fast +TARGET_COMMITTEE_SIZE: 4 +# 2**11 (= 2,048) +MAX_VALIDATORS_PER_COMMITTEE: 2048 +# [customized] Faster, but unsecure. +SHUFFLE_ROUND_COUNT: 10 +# 4 +HYSTERESIS_QUOTIENT: 4 +# 1 (minus 0.25) +HYSTERESIS_DOWNWARD_MULTIPLIER: 1 +# 5 (plus 1.25) +HYSTERESIS_UPWARD_MULTIPLIER: 5 -# Networking -# --------------------------------------------------------------- -# `10 * 2**20` (= 10485760, 10 MiB) -GOSSIP_MAX_SIZE: 10485760 -# `2**10` (= 1024) -MAX_REQUEST_BLOCKS: 1024 -# `2**8` (= 256) -EPOCHS_PER_SUBNET_SUBSCRIPTION: 256 -# [customized] `MIN_VALIDATOR_WITHDRAWABILITY_DELAY + CHURN_LIMIT_QUOTIENT // 2` (= 272) -MIN_EPOCHS_FOR_BLOCK_REQUESTS: 272 -# `10 * 2**20` (=10485760, 10 MiB) -MAX_CHUNK_SIZE: 10485760 -# 5s -TTFB_TIMEOUT: 5 -# 10s -RESP_TIMEOUT: 10 -ATTESTATION_PROPAGATION_SLOT_RANGE: 32 -# 500ms -MAXIMUM_GOSSIP_CLOCK_DISPARITY: 500 -MESSAGE_DOMAIN_INVALID_SNAPPY: 0x00000000 -MESSAGE_DOMAIN_VALID_SNAPPY: 0x01000000 -# 2 subnets per node -SUBNETS_PER_NODE: 2 -# 2**8 (= 64) -ATTESTATION_SUBNET_COUNT: 64 -ATTESTATION_SUBNET_EXTRA_BITS: 0 -# ceillog2(ATTESTATION_SUBNET_COUNT) + ATTESTATION_SUBNET_EXTRA_BITS -ATTESTATION_SUBNET_PREFIX_BITS: 6 - -# Deneb -# `2**7` (=128) -MAX_REQUEST_BLOCKS_DENEB: 128 -# MAX_REQUEST_BLOCKS_DENEB * MAX_BLOBS_PER_BLOCK -MAX_REQUEST_BLOB_SIDECARS: 768 -# `2**12` (= 4096 epochs, ~18 days) -MIN_EPOCHS_FOR_BLOB_SIDECARS_REQUESTS: 4096 -# `6` -BLOB_SIDECAR_SUBNET_COUNT: 6 \ No newline at end of file + +# Fork Choice +# --------------------------------------------------------------- +# 2**1 (= 1) +SAFE_SLOTS_TO_UPDATE_JUSTIFIED: 2 + + +# Gwei values +# --------------------------------------------------------------- +# 2**0 * 10**9 (= 1,000,000,000) Gwei +MIN_DEPOSIT_AMOUNT: 1000000000 +# 2**5 * 10**9 (= 32,000,000,000) Gwei +MAX_EFFECTIVE_BALANCE: 32000000000 +# 2**0 * 10**9 (= 1,000,000,000) Gwei +EFFECTIVE_BALANCE_INCREMENT: 1000000000 + + +# Time parameters +# --------------------------------------------------------------- +# 2**0 (= 1) slots 6 seconds +MIN_ATTESTATION_INCLUSION_DELAY: 1 +# [customized] fast epochs +SLOTS_PER_EPOCH: 8 +# 2**0 (= 1) epochs +MIN_SEED_LOOKAHEAD: 1 +# 2**2 (= 4) epochs +MAX_SEED_LOOKAHEAD: 4 +# [customized] higher frequency new deposits from eth1 for testing +EPOCHS_PER_ETH1_VOTING_PERIOD: 4 +# [customized] smaller state +SLOTS_PER_HISTORICAL_ROOT: 64 +# 2**2 (= 4) epochs +MIN_EPOCHS_TO_INACTIVITY_PENALTY: 4 + + +# State list lengths +# --------------------------------------------------------------- +# [customized] smaller state +EPOCHS_PER_HISTORICAL_VECTOR: 64 +# [customized] smaller state +EPOCHS_PER_SLASHINGS_VECTOR: 64 +# 2**24 (= 16,777,216) historical roots +HISTORICAL_ROOTS_LIMIT: 16777216 +# 2**40 (= 1,099,511,627,776) validator spots +VALIDATOR_REGISTRY_LIMIT: 1099511627776 + + +# Reward and penalty quotients +# --------------------------------------------------------------- +# 2**6 (= 64) +BASE_REWARD_FACTOR: 64 +# 2**9 (= 512) +WHISTLEBLOWER_REWARD_QUOTIENT: 512 +# 2**3 (= 8) +PROPOSER_REWARD_QUOTIENT: 8 +# [customized] 2**25 (= 33,554,432) +INACTIVITY_PENALTY_QUOTIENT: 33554432 +# [customized] 2**6 (= 64) +MIN_SLASHING_PENALTY_QUOTIENT: 64 +# [customized] 2 (lower safety margin than Phase 0 genesis but different than mainnet config for testing) +PROPORTIONAL_SLASHING_MULTIPLIER: 2 + + +# Max operations per block +# --------------------------------------------------------------- +# 2**4 (= 16) +MAX_PROPOSER_SLASHINGS: 16 +# 2**1 (= 2) +MAX_ATTESTER_SLASHINGS: 2 +# 2**7 (= 128) +MAX_ATTESTATIONS: 128 +# 2**4 (= 16) +MAX_DEPOSITS: 16 +# 2**4 (= 16) +MAX_VOLUNTARY_EXITS: 16 +# Minimal preset - Sharding + +# Misc +# --------------------------------------------------------------- +# Misc +# [customized] reduced for testing +MAX_SHARDS: 8 +# [customized] reduced for testing +INITIAL_ACTIVE_SHARDS: 2 +# 2**3 (= 8) +SAMPLE_PRICE_ADJUSTMENT_COEFFICIENT: 8 +# [customized] reduced for testing +MAX_SHARD_PROPOSER_SLASHINGS: 4 +# +MAX_SHARD_HEADERS_PER_SHARD: 4 +# 2**8 (= 256) +SHARD_STATE_MEMORY_SLOTS: 256 +# 2**40 (= 1,099,511,627,776) +BLOB_BUILDER_REGISTRY_LIMIT: 1099511627776 + +# Shard blob samples +# --------------------------------------------------------------- +# 2**11 (= 2,048) +MAX_SAMPLES_PER_BLOCK: 2048 +# 2**10 (= 1,1024) +TARGET_SAMPLES_PER_BLOCK: 1024 + +# Gwei values +# --------------------------------------------------------------- +# 2**33 (= 8,589,934,592) Gwei +MAX_SAMPLE_PRICE: 8589934592 +# 2**3 (= 8) Gwei +MIN_SAMPLE_PRICE: 8 diff --git a/config/phase0.yml b/config/phase0.yml new file mode 100644 index 0000000000..85a5ad7a96 --- /dev/null +++ b/config/phase0.yml @@ -0,0 +1,171 @@ +# Mainnet preset + +CONFIG_NAME: "mainnet" + +# Misc +# --------------------------------------------------------------- +# 2**6 (= 64) +MAX_COMMITTEES_PER_SLOT: 64 +# 2**7 (= 128) +TARGET_COMMITTEE_SIZE: 128 +# 2**11 (= 2,048) +MAX_VALIDATORS_PER_COMMITTEE: 2048 +# 2**2 (= 4) +MIN_PER_EPOCH_CHURN_LIMIT: 4 +# 2**16 (= 65,536) +CHURN_LIMIT_QUOTIENT: 65536 +# See issue 563 +SHUFFLE_ROUND_COUNT: 90 +# `2**14` (= 16,384) +MIN_GENESIS_ACTIVE_VALIDATOR_COUNT: 16384 +# Dec 1, 2020, 12pm UTC +MIN_GENESIS_TIME: 1606824000 +# 4 +HYSTERESIS_QUOTIENT: 4 +# 1 (minus 0.25) +HYSTERESIS_DOWNWARD_MULTIPLIER: 1 +# 5 (plus 1.25) +HYSTERESIS_UPWARD_MULTIPLIER: 5 + + +# Fork Choice +# --------------------------------------------------------------- +# 2**3 (= 8) +SAFE_SLOTS_TO_UPDATE_JUSTIFIED: 8 + + +# Validator +# --------------------------------------------------------------- +# 2**11 (= 2,048) +ETH1_FOLLOW_DISTANCE: 2048 +# 2**4 (= 16) +TARGET_AGGREGATORS_PER_COMMITTEE: 16 +# 2**0 (= 1) +RANDOM_SUBNETS_PER_VALIDATOR: 1 +# 2**8 (= 256) +EPOCHS_PER_RANDOM_SUBNET_SUBSCRIPTION: 256 +# 14 (estimate from Eth1 mainnet) +SECONDS_PER_ETH1_BLOCK: 14 + + +# Deposit contract +# --------------------------------------------------------------- +# Ethereum PoW Mainnet +DEPOSIT_CHAIN_ID: 1 +DEPOSIT_NETWORK_ID: 1 +# **TBD** +DEPOSIT_CONTRACT_ADDRESS: 0x00000000219ab540356cBB839Cbe05303d7705Fa + + +# Gwei values +# --------------------------------------------------------------- +# 2**0 * 10**9 (= 1,000,000,000) Gwei +MIN_DEPOSIT_AMOUNT: 1000000000 +# 2**5 * 10**9 (= 32,000,000,000) Gwei +MAX_EFFECTIVE_BALANCE: 32000000000 +# 2**4 * 10**9 (= 16,000,000,000) Gwei +EJECTION_BALANCE: 16000000000 +# 2**0 * 10**9 (= 1,000,000,000) Gwei +EFFECTIVE_BALANCE_INCREMENT: 1000000000 + + +# Initial values +# --------------------------------------------------------------- +# Mainnet initial fork version, recommend altering for testnets +GENESIS_FORK_VERSION: 0x00000000 +BLS_WITHDRAWAL_PREFIX: 0x00 + + +# Time parameters +# --------------------------------------------------------------- +# 604800 seconds (7 days) +GENESIS_DELAY: 604800 +# 12 seconds +SECONDS_PER_SLOT: 12 +# 2**0 (= 1) slots 12 seconds +MIN_ATTESTATION_INCLUSION_DELAY: 1 +# 2**5 (= 32) slots 6.4 minutes +SLOTS_PER_EPOCH: 32 +# 2**0 (= 1) epochs 6.4 minutes +MIN_SEED_LOOKAHEAD: 1 +# 2**2 (= 4) epochs 25.6 minutes +MAX_SEED_LOOKAHEAD: 4 +# 2**6 (= 64) epochs ~6.8 hours +EPOCHS_PER_ETH1_VOTING_PERIOD: 64 +# 2**13 (= 8,192) slots ~13 hours +SLOTS_PER_HISTORICAL_ROOT: 8192 +# 2**8 (= 256) epochs ~27 hours +MIN_VALIDATOR_WITHDRAWABILITY_DELAY: 256 +# 2**8 (= 256) epochs ~27 hours +SHARD_COMMITTEE_PERIOD: 256 +# 2**2 (= 4) epochs 25.6 minutes +MIN_EPOCHS_TO_INACTIVITY_PENALTY: 4 + + +# State vector lengths +# --------------------------------------------------------------- +# 2**16 (= 65,536) epochs ~0.8 years +EPOCHS_PER_HISTORICAL_VECTOR: 65536 +# 2**13 (= 8,192) epochs ~36 days +EPOCHS_PER_SLASHINGS_VECTOR: 8192 +# 2**24 (= 16,777,216) historical roots, ~26,131 years +HISTORICAL_ROOTS_LIMIT: 16777216 +# 2**40 (= 1,099,511,627,776) validator spots +VALIDATOR_REGISTRY_LIMIT: 1099511627776 + + +# Reward and penalty quotients +# --------------------------------------------------------------- +# 2**6 (= 64) +BASE_REWARD_FACTOR: 64 +# 2**9 (= 512) +WHISTLEBLOWER_REWARD_QUOTIENT: 512 +# 2**3 (= 8) +PROPOSER_REWARD_QUOTIENT: 8 +# 2**26 (= 67,108,864) +INACTIVITY_PENALTY_QUOTIENT: 67108864 +# 2**7 (= 128) (lower safety margin at Phase 0 genesis) +MIN_SLASHING_PENALTY_QUOTIENT: 128 +# 1 (lower safety margin at Phase 0 genesis) +PROPORTIONAL_SLASHING_MULTIPLIER: 1 + + +# Max operations per block +# --------------------------------------------------------------- +# 2**4 (= 16) +MAX_PROPOSER_SLASHINGS: 16 +# 2**1 (= 2) +MAX_ATTESTER_SLASHINGS: 2 +# 2**7 (= 128) +MAX_ATTESTATIONS: 128 +# 2**4 (= 16) +MAX_DEPOSITS: 16 +# 2**4 (= 16) +MAX_VOLUNTARY_EXITS: 16 + + +# Signature domains +# --------------------------------------------------------------- +DOMAIN_BEACON_PROPOSER: 0x00000000 +DOMAIN_BEACON_ATTESTER: 0x01000000 +DOMAIN_RANDAO: 0x02000000 +DOMAIN_DEPOSIT: 0x03000000 +DOMAIN_VOLUNTARY_EXIT: 0x04000000 +DOMAIN_SELECTION_PROOF: 0x05000000 +DOMAIN_AGGREGATE_AND_PROOF: 0x06000000 + +# Max operations per block +# --------------------------------------------------------------- +# 2**4 (= 16) +MAX_BLS_TO_EXECUTION_CHANGES: 16 + + +# Execution +# --------------------------------------------------------------- +# [customized] 2**2 (= 4) +MAX_WITHDRAWALS_PER_PAYLOAD: 16 + +# Withdrawals processing +# --------------------------------------------------------------- +# [customized] 2**4 (= 16) validators +MAX_VALIDATORS_PER_WITHDRAWALS_SWEEP: 16384 \ No newline at end of file diff --git a/config/prater.chain.yml b/config/prater.chain.yml index a24c0754a3..e6d0c96f5c 100644 --- a/config/prater.chain.yml +++ b/config/prater.chain.yml @@ -1,11 +1,8 @@ -# https://github.com/eth-clients/goerli/blob/d6a227e/prater/config.yaml - # Prater config # Extends the mainnet preset -PRESET_BASE: 'mainnet' - -CONFIG_NAME: 'prater' +PRESET_BASE: "mainnet" +CONFIG_NAME: "prater" # Transition # --------------------------------------------------------------- @@ -15,7 +12,6 @@ TERMINAL_TOTAL_DIFFICULTY: 10790000 TERMINAL_BLOCK_HASH: 0x0000000000000000000000000000000000000000000000000000000000000000 TERMINAL_BLOCK_HASH_ACTIVATION_EPOCH: 18446744073709551615 - # Genesis # --------------------------------------------------------------- # `2**14` (= 16,384) @@ -27,7 +23,6 @@ GENESIS_FORK_VERSION: 0x00001020 # Customized for Prater: 1919188 seconds (Mar-23-2021 02:00:00 PM +UTC) GENESIS_DELAY: 1919188 - # Forking # --------------------------------------------------------------- # Some forks are disabled for now: @@ -43,9 +38,13 @@ BELLATRIX_FORK_EPOCH: 112260 # Capella CAPELLA_FORK_VERSION: 0x03001020 CAPELLA_FORK_EPOCH: 162304 -# DENEB +# Sharding DENEB_FORK_VERSION: 0x04001020 DENEB_FORK_EPOCH: 18446744073709551615 +# Byzantium +BYZANTIUM_FORK_BLOCK: 0 +# Constantinople +CONSTANTINOPLE_FORK_BLOCK: 0 # Time parameters # --------------------------------------------------------------- @@ -60,7 +59,6 @@ SHARD_COMMITTEE_PERIOD: 256 # 2**11 (= 2,048) Eth1 blocks ~8 hours ETH1_FOLLOW_DISTANCE: 2048 - # Validator cycle # --------------------------------------------------------------- # 2**2 (= 4) @@ -74,11 +72,225 @@ MIN_PER_EPOCH_CHURN_LIMIT: 4 # 2**16 (= 65,536) CHURN_LIMIT_QUOTIENT: 65536 - # Deposit contract # --------------------------------------------------------------- # Ethereum Goerli testnet DEPOSIT_CHAIN_ID: 5 DEPOSIT_NETWORK_ID: 5 # Prater test deposit contract on Goerli Testnet -DEPOSIT_CONTRACT_ADDRESS: 0xff50ed3d0ec03aC01D4C79aAd74928BFF48a7b2b \ No newline at end of file +DEPOSIT_CONTRACT_ADDRESS: 0xff50ed3d0ec03aC01D4C79aAd74928BFF48a7b2b + +# Mainnet preset - Altair + +# Updated penalty values +# --------------------------------------------------------------- +# 3 * 2**24 (= 50,331,648) +INACTIVITY_PENALTY_QUOTIENT_ALTAIR: 50331648 +# 2**6 (= 64) +MIN_SLASHING_PENALTY_QUOTIENT_ALTAIR: 64 +# 2 +PROPORTIONAL_SLASHING_MULTIPLIER_ALTAIR: 2 + +# Sync committee +# --------------------------------------------------------------- +# 2**9 (= 512) +SYNC_COMMITTEE_SIZE: 512 +# 2**8 (= 256) +EPOCHS_PER_SYNC_COMMITTEE_PERIOD: 256 + +# Sync protocol +# --------------------------------------------------------------- +# 1 +MIN_SYNC_COMMITTEE_PARTICIPANTS: 1 +# SLOTS_PER_EPOCH * EPOCHS_PER_SYNC_COMMITTEE_PERIOD (= 32 * 256) +UPDATE_TIMEOUT: 8192 +# Mainnet preset - Bellatrix + +# Updated penalty values +# --------------------------------------------------------------- +# 2**24 (= 16,777,216) +INACTIVITY_PENALTY_QUOTIENT_BELLATRIX: 16777216 +# 2**5 (= 32) +MIN_SLASHING_PENALTY_QUOTIENT_BELLATRIX: 32 +# 3 +PROPORTIONAL_SLASHING_MULTIPLIER_BELLATRIX: 3 + +# Execution +# --------------------------------------------------------------- +# 2**30 (= 1,073,741,824) +MAX_BYTES_PER_TRANSACTION: 1073741824 +# 2**20 (= 1,048,576) +MAX_TRANSACTIONS_PER_PAYLOAD: 1048576 +# 2**8 (= 256) +BYTES_PER_LOGS_BLOOM: 256 +# 2**5 (= 32) +MAX_EXTRA_DATA_BYTES: 32 +# Minimal preset - Capella +# Mainnet preset - Custody Game + +# Time parameters +# --------------------------------------------------------------- +# 2**1 (= 2) epochs, 12.8 minutes +RANDAO_PENALTY_EPOCHS: 2 +# 2**15 (= 32,768) epochs, ~146 days +EARLY_DERIVED_SECRET_PENALTY_MAX_FUTURE_EPOCHS: 32768 +# 2**14 (= 16,384) epochs ~73 days +EPOCHS_PER_CUSTODY_PERIOD: 16384 +# 2**11 (= 2,048) epochs, ~9 days +CUSTODY_PERIOD_TO_RANDAO_PADDING: 2048 +# 2**15 (= 32,768) epochs, ~146 days +MAX_CHUNK_CHALLENGE_DELAY: 32768 + +# Max operations +# --------------------------------------------------------------- +# 2**8 (= 256) +MAX_CUSTODY_KEY_REVEALS: 256 +# 2**0 (= 1) +MAX_EARLY_DERIVED_SECRET_REVEALS: 1 +# 2**2 (= 2) +MAX_CUSTODY_CHUNK_CHALLENGES: 4 +# 2** 4 (= 16) +MAX_CUSTODY_CHUNK_CHALLENGE_RESP: 16 +# 2**0 (= 1) +MAX_CUSTODY_SLASHINGS: 1 + +# Reward and penalty quotients +# --------------------------------------------------------------- +EARLY_DERIVED_SECRET_REVEAL_SLOT_REWARD_MULTIPLE: 2 +# 2**8 (= 256) +MINOR_REWARD_QUOTIENT: 256 +# Mainnet preset - Phase0 + +# Misc +# --------------------------------------------------------------- +# 2**6 (= 64) +MAX_COMMITTEES_PER_SLOT: 64 +# 2**7 (= 128) +TARGET_COMMITTEE_SIZE: 128 +# 2**11 (= 2,048) +MAX_VALIDATORS_PER_COMMITTEE: 2048 +# See issue 563 +SHUFFLE_ROUND_COUNT: 90 +# 4 +HYSTERESIS_QUOTIENT: 4 +# 1 (minus 0.25) +HYSTERESIS_DOWNWARD_MULTIPLIER: 1 +# 5 (plus 1.25) +HYSTERESIS_UPWARD_MULTIPLIER: 5 + +# Fork Choice +# --------------------------------------------------------------- +# 2**3 (= 8) +SAFE_SLOTS_TO_UPDATE_JUSTIFIED: 8 + +# Gwei values +# --------------------------------------------------------------- +# 2**0 * 10**9 (= 1,000,000,000) Gwei +MIN_DEPOSIT_AMOUNT: 1000000000 +# 2**5 * 10**9 (= 32,000,000,000) Gwei +MAX_EFFECTIVE_BALANCE: 32000000000 +# 2**0 * 10**9 (= 1,000,000,000) Gwei +EFFECTIVE_BALANCE_INCREMENT: 1000000000 + +# Time parameters +# --------------------------------------------------------------- +# 2**0 (= 1) slots 12 seconds +MIN_ATTESTATION_INCLUSION_DELAY: 1 +# 2**5 (= 32) slots 6.4 minutes +SLOTS_PER_EPOCH: 32 +# 2**0 (= 1) epochs 6.4 minutes +MIN_SEED_LOOKAHEAD: 1 +# 2**2 (= 4) epochs 25.6 minutes +MAX_SEED_LOOKAHEAD: 4 +# 2**6 (= 64) epochs ~6.8 hours +EPOCHS_PER_ETH1_VOTING_PERIOD: 64 +# 2**13 (= 8,192) slots ~27 hours +SLOTS_PER_HISTORICAL_ROOT: 8192 +# 2**2 (= 4) epochs 25.6 minutes +MIN_EPOCHS_TO_INACTIVITY_PENALTY: 4 + +# State list lengths +# --------------------------------------------------------------- +# 2**16 (= 65,536) epochs ~0.8 years +EPOCHS_PER_HISTORICAL_VECTOR: 65536 +# 2**13 (= 8,192) epochs ~36 days +EPOCHS_PER_SLASHINGS_VECTOR: 8192 +# 2**24 (= 16,777,216) historical roots, ~26,131 years +HISTORICAL_ROOTS_LIMIT: 16777216 +# 2**40 (= 1,099,511,627,776) validator spots +VALIDATOR_REGISTRY_LIMIT: 1099511627776 + +# Reward and penalty quotients +# --------------------------------------------------------------- +# 2**6 (= 64) +BASE_REWARD_FACTOR: 64 +# 2**9 (= 512) +WHISTLEBLOWER_REWARD_QUOTIENT: 512 +# 2**3 (= 8) +PROPOSER_REWARD_QUOTIENT: 8 +# 2**26 (= 67,108,864) +INACTIVITY_PENALTY_QUOTIENT: 67108864 +# 2**7 (= 128) (lower safety margin at Phase 0 genesis) +MIN_SLASHING_PENALTY_QUOTIENT: 128 +# 1 (lower safety margin at Phase 0 genesis) +PROPORTIONAL_SLASHING_MULTIPLIER: 1 + +# Max operations per block +# --------------------------------------------------------------- +# 2**4 (= 16) +MAX_PROPOSER_SLASHINGS: 16 +# 2**1 (= 2) +MAX_ATTESTER_SLASHINGS: 2 +# 2**7 (= 128) +MAX_ATTESTATIONS: 128 +# 2**4 (= 16) +MAX_DEPOSITS: 16 +# 2**4 (= 16) +MAX_VOLUNTARY_EXITS: 16 +# Mainnet preset - Sharding + +# Misc +# --------------------------------------------------------------- +# 2**10 (= 1,024) +MAX_SHARDS: 1024 +# 2**6 (= 64) +INITIAL_ACTIVE_SHARDS: 64 +# 2**3 (= 8) +SAMPLE_PRICE_ADJUSTMENT_COEFFICIENT: 8 +# 2**4 (= 16) +MAX_SHARD_PROPOSER_SLASHINGS: 16 +# +MAX_SHARD_HEADERS_PER_SHARD: 4 +# 2**8 (= 256) +SHARD_STATE_MEMORY_SLOTS: 256 +# 2**40 (= 1,099,511,627,776) +BLOB_BUILDER_REGISTRY_LIMIT: 1099511627776 + +# Shard blob samples +# --------------------------------------------------------------- +# 2**11 (= 2,048) +MAX_SAMPLES_PER_BLOCK: 2048 +# 2**10 (= 1,1024) +TARGET_SAMPLES_PER_BLOCK: 1024 + +# Gwei values +# --------------------------------------------------------------- +# 2**33 (= 8,589,934,592) Gwei +MAX_SAMPLE_PRICE: 8589934592 +# 2**3 (= 8) Gwei +MIN_SAMPLE_PRICE: 8 + +# Max operations per block +# --------------------------------------------------------------- +# 2**4 (= 16) +MAX_BLS_TO_EXECUTION_CHANGES: 16 + +# Execution +# --------------------------------------------------------------- +# [customized] 2**2 (= 4) +MAX_WITHDRAWALS_PER_PAYLOAD: 16 + +# Withdrawals processing +# --------------------------------------------------------------- +# [customized] 2**4 (= 16) validators +MAX_VALIDATORS_PER_WITHDRAWALS_SWEEP: 16384 diff --git a/config/prater0.yml b/config/prater0.yml new file mode 100644 index 0000000000..d62ab45183 --- /dev/null +++ b/config/prater0.yml @@ -0,0 +1,171 @@ +# Prater preset + +CONFIG_NAME: "prater" + +# Misc +# --------------------------------------------------------------- +# 2**6 (= 64) +MAX_COMMITTEES_PER_SLOT: 64 +# 2**7 (= 128) +TARGET_COMMITTEE_SIZE: 128 +# 2**11 (= 2,048) +MAX_VALIDATORS_PER_COMMITTEE: 2048 +# 2**2 (= 4) +MIN_PER_EPOCH_CHURN_LIMIT: 4 +# 2**16 (= 65,536) +CHURN_LIMIT_QUOTIENT: 65536 +# See issue 563 +SHUFFLE_ROUND_COUNT: 90 +# `2**14` (= 16,384) +MIN_GENESIS_ACTIVE_VALIDATOR_COUNT: 16384 +# Mar-01-2021 08:53:32 AM +UTC +MIN_GENESIS_TIME: 1614588812 +# 4 +HYSTERESIS_QUOTIENT: 4 +# 1 (minus 0.25) +HYSTERESIS_DOWNWARD_MULTIPLIER: 1 +# 5 (plus 1.25) +HYSTERESIS_UPWARD_MULTIPLIER: 5 + + +# Fork Choice +# --------------------------------------------------------------- +# 2**3 (= 8) +SAFE_SLOTS_TO_UPDATE_JUSTIFIED: 8 + + +# Validator +# --------------------------------------------------------------- +# 2**11 (= 2,048) +ETH1_FOLLOW_DISTANCE: 2048 +# 2**4 (= 16) +TARGET_AGGREGATORS_PER_COMMITTEE: 16 +# 2**0 (= 1) +RANDOM_SUBNETS_PER_VALIDATOR: 1 +# 2**8 (= 256) +EPOCHS_PER_RANDOM_SUBNET_SUBSCRIPTION: 256 +# 14 (estimate from Eth1 mainnet) +SECONDS_PER_ETH1_BLOCK: 14 + + +# Deposit contract +# --------------------------------------------------------------- +# Ethereum Goerli testnet +DEPOSIT_CHAIN_ID: 5 +DEPOSIT_NETWORK_ID: 5 +# Prater test deposit contract on Goerli Testnet +DEPOSIT_CONTRACT_ADDRESS: 0xff50ed3d0ec03aC01D4C79aAd74928BFF48a7b2b + + +# Gwei values +# --------------------------------------------------------------- +# 2**0 * 10**9 (= 1,000,000,000) Gwei +MIN_DEPOSIT_AMOUNT: 1000000000 +# 2**5 * 10**9 (= 32,000,000,000) Gwei +MAX_EFFECTIVE_BALANCE: 32000000000 +# 2**4 * 10**9 (= 16,000,000,000) Gwei +EJECTION_BALANCE: 16000000000 +# 2**0 * 10**9 (= 1,000,000,000) Gwei +EFFECTIVE_BALANCE_INCREMENT: 1000000000 + + +# Initial values +# --------------------------------------------------------------- +# Prater area code (Vienna) +GENESIS_FORK_VERSION: 0x00001020 +BLS_WITHDRAWAL_PREFIX: 0x00 + + +# Time parameters +# --------------------------------------------------------------- +# Customized for Prater: 1919188 seconds (Mar-23-2021 02:00:00 PM +UTC) +GENESIS_DELAY: 1919188 +# 12 seconds +SECONDS_PER_SLOT: 12 +# 2**0 (= 1) slots 12 seconds +MIN_ATTESTATION_INCLUSION_DELAY: 1 +# 2**5 (= 32) slots 6.4 minutes +SLOTS_PER_EPOCH: 32 +# 2**0 (= 1) epochs 6.4 minutes +MIN_SEED_LOOKAHEAD: 1 +# 2**2 (= 4) epochs 25.6 minutes +MAX_SEED_LOOKAHEAD: 4 +# 2**6 (= 64) epochs ~6.8 hours +EPOCHS_PER_ETH1_VOTING_PERIOD: 64 +# 2**13 (= 8,192) slots ~13 hours +SLOTS_PER_HISTORICAL_ROOT: 8192 +# 2**8 (= 256) epochs ~27 hours +MIN_VALIDATOR_WITHDRAWABILITY_DELAY: 256 +# 2**8 (= 256) epochs ~27 hours +SHARD_COMMITTEE_PERIOD: 256 +# 2**2 (= 4) epochs 25.6 minutes +MIN_EPOCHS_TO_INACTIVITY_PENALTY: 4 + + +# State vector lengths +# --------------------------------------------------------------- +# 2**16 (= 65,536) epochs ~0.8 years +EPOCHS_PER_HISTORICAL_VECTOR: 65536 +# 2**13 (= 8,192) epochs ~36 days +EPOCHS_PER_SLASHINGS_VECTOR: 8192 +# 2**24 (= 16,777,216) historical roots, ~26,131 years +HISTORICAL_ROOTS_LIMIT: 16777216 +# 2**40 (= 1,099,511,627,776) validator spots +VALIDATOR_REGISTRY_LIMIT: 1099511627776 + + +# Reward and penalty quotients +# --------------------------------------------------------------- +# 2**6 (= 64) +BASE_REWARD_FACTOR: 64 +# 2**9 (= 512) +WHISTLEBLOWER_REWARD_QUOTIENT: 512 +# 2**3 (= 8) +PROPOSER_REWARD_QUOTIENT: 8 +# 2**26 (= 67,108,864) +INACTIVITY_PENALTY_QUOTIENT: 67108864 +# 2**7 (= 128) (lower safety margin at Phase 0 genesis) +MIN_SLASHING_PENALTY_QUOTIENT: 128 +# 1 (lower safety margin at Phase 0 genesis) +PROPORTIONAL_SLASHING_MULTIPLIER: 1 + + +# Max operations per block +# --------------------------------------------------------------- +# 2**4 (= 16) +MAX_PROPOSER_SLASHINGS: 16 +# 2**1 (= 2) +MAX_ATTESTER_SLASHINGS: 2 +# 2**7 (= 128) +MAX_ATTESTATIONS: 128 +# 2**4 (= 16) +MAX_DEPOSITS: 16 +# 2**4 (= 16) +MAX_VOLUNTARY_EXITS: 16 + + +# Signature domains +# --------------------------------------------------------------- +DOMAIN_BEACON_PROPOSER: 0x00000000 +DOMAIN_BEACON_ATTESTER: 0x01000000 +DOMAIN_RANDAO: 0x02000000 +DOMAIN_DEPOSIT: 0x03000000 +DOMAIN_VOLUNTARY_EXIT: 0x04000000 +DOMAIN_SELECTION_PROOF: 0x05000000 +DOMAIN_AGGREGATE_AND_PROOF: 0x06000000 + +# Max operations per block +# --------------------------------------------------------------- +# 2**4 (= 16) +MAX_BLS_TO_EXECUTION_CHANGES: 16 + + +# Execution +# --------------------------------------------------------------- +# [customized] 2**2 (= 4) +MAX_WITHDRAWALS_PER_PAYLOAD: 16 + +# Withdrawals processing +# --------------------------------------------------------------- +# [customized] 2**4 (= 16) validators +MAX_VALIDATORS_PER_WITHDRAWALS_SWEEP: 16384 \ No newline at end of file diff --git a/config/ropsten.chain.yml b/config/ropsten.chain.yml new file mode 100644 index 0000000000..391f4ae7f6 --- /dev/null +++ b/config/ropsten.chain.yml @@ -0,0 +1,302 @@ +# Extends the mainnet preset +PRESET_BASE: 'mainnet' +CONFIG_NAME: 'ropsten' + +# Genesis +# --------------------------------------------------------------- +MIN_GENESIS_ACTIVE_VALIDATOR_COUNT: 100000 +# Monday, May 30th, 2022 3:00:00 PM +UTC +MIN_GENESIS_TIME: 1653318000 +GENESIS_FORK_VERSION: 0x80000069 +GENESIS_DELAY: 604800 + + +# Forking +# --------------------------------------------------------------- +# Some forks are disabled for now: +# - These may be re-assigned to another fork-version later +# - Temporarily set to max uint64 value: 2**64 - 1 + +# Altair +ALTAIR_FORK_VERSION: 0x80000070 +ALTAIR_FORK_EPOCH: 500 + +# Merge +BELLATRIX_FORK_VERSION: 0x80000071 +BELLATRIX_FORK_EPOCH: 750 +TERMINAL_TOTAL_DIFFICULTY: 50000000000000000 +TERMINAL_BLOCK_HASH: 0x0000000000000000000000000000000000000000000000000000000000000000 +TERMINAL_BLOCK_HASH_ACTIVATION_EPOCH: 18446744073709551615 + +# Capella +CAPELLA_FORK_VERSION: 0x03001020 +CAPELLA_FORK_EPOCH: 18446744073709551615 + +# Sharding +SHARDING_FORK_VERSION: 0x04001020 +SHARDING_FORK_EPOCH: 18446744073709551615 + +# Time parameters +# --------------------------------------------------------------- +# 12 seconds +SECONDS_PER_SLOT: 12 +# 14 (estimate from Eth1 mainnet) +SECONDS_PER_ETH1_BLOCK: 14 +# 2**8 (= 256) epochs ~27 hours +MIN_VALIDATOR_WITHDRAWABILITY_DELAY: 256 +# 2**8 (= 256) epochs ~27 hours +SHARD_COMMITTEE_PERIOD: 256 +# 2**11 (= 2,048) Eth1 blocks ~8 hours +ETH1_FOLLOW_DISTANCE: 2048 + + +# Validator cycle +# --------------------------------------------------------------- +# 2**2 (= 4) +INACTIVITY_SCORE_BIAS: 4 +# 2**4 (= 16) +INACTIVITY_SCORE_RECOVERY_RATE: 16 +# 2**4 * 10**9 (= 16,000,000,000) Gwei +EJECTION_BALANCE: 16000000000 +# 2**2 (= 4) +MIN_PER_EPOCH_CHURN_LIMIT: 4 +# 2**16 (= 65,536) +CHURN_LIMIT_QUOTIENT: 65536 + + +# Fork choice +# --------------------------------------------------------------- +# 40% +PROPOSER_SCORE_BOOST: 40 + +# Deposit contract +# --------------------------------------------------------------- +DEPOSIT_CHAIN_ID: 3 +DEPOSIT_NETWORK_ID: 3 +DEPOSIT_CONTRACT_ADDRESS: 0x6f22fFbC56eFF051aECF839396DD1eD9aD6BBA9D + +# Mainnet preset - Altair + +# Updated penalty values +# --------------------------------------------------------------- +# 3 * 2**24 (= 50,331,648) +INACTIVITY_PENALTY_QUOTIENT_ALTAIR: 50331648 +# 2**6 (= 64) +MIN_SLASHING_PENALTY_QUOTIENT_ALTAIR: 64 +# 2 +PROPORTIONAL_SLASHING_MULTIPLIER_ALTAIR: 2 + + +# Sync committee +# --------------------------------------------------------------- +# 2**9 (= 512) +SYNC_COMMITTEE_SIZE: 512 +# 2**8 (= 256) +EPOCHS_PER_SYNC_COMMITTEE_PERIOD: 256 + + +# Sync protocol +# --------------------------------------------------------------- +# 1 +MIN_SYNC_COMMITTEE_PARTICIPANTS: 1 +# SLOTS_PER_EPOCH * EPOCHS_PER_SYNC_COMMITTEE_PERIOD (= 32 * 256) +UPDATE_TIMEOUT: 8192 +# Mainnet preset - Bellatrix + +# Updated penalty values +# --------------------------------------------------------------- +# 2**24 (= 16,777,216) +INACTIVITY_PENALTY_QUOTIENT_BELLATRIX: 16777216 +# 2**5 (= 32) +MIN_SLASHING_PENALTY_QUOTIENT_BELLATRIX: 32 +# 3 +PROPORTIONAL_SLASHING_MULTIPLIER_BELLATRIX: 3 + +# Execution +# --------------------------------------------------------------- +# 2**30 (= 1,073,741,824) +MAX_BYTES_PER_TRANSACTION: 1073741824 +# 2**20 (= 1,048,576) +MAX_TRANSACTIONS_PER_PAYLOAD: 1048576 +# 2**8 (= 256) +BYTES_PER_LOGS_BLOOM: 256 +# 2**5 (= 32) +MAX_EXTRA_DATA_BYTES: 32 +# Minimal preset - Capella +# Mainnet preset - Custody Game + +# Time parameters +# --------------------------------------------------------------- +# 2**1 (= 2) epochs, 12.8 minutes +RANDAO_PENALTY_EPOCHS: 2 +# 2**15 (= 32,768) epochs, ~146 days +EARLY_DERIVED_SECRET_PENALTY_MAX_FUTURE_EPOCHS: 32768 +# 2**14 (= 16,384) epochs ~73 days +EPOCHS_PER_CUSTODY_PERIOD: 16384 +# 2**11 (= 2,048) epochs, ~9 days +CUSTODY_PERIOD_TO_RANDAO_PADDING: 2048 +# 2**15 (= 32,768) epochs, ~146 days +MAX_CHUNK_CHALLENGE_DELAY: 32768 + + +# Max operations +# --------------------------------------------------------------- +# 2**8 (= 256) +MAX_CUSTODY_KEY_REVEALS: 256 +# 2**0 (= 1) +MAX_EARLY_DERIVED_SECRET_REVEALS: 1 +# 2**2 (= 2) +MAX_CUSTODY_CHUNK_CHALLENGES: 4 +# 2** 4 (= 16) +MAX_CUSTODY_CHUNK_CHALLENGE_RESP: 16 +# 2**0 (= 1) +MAX_CUSTODY_SLASHINGS: 1 + + +# Reward and penalty quotients +# --------------------------------------------------------------- +EARLY_DERIVED_SECRET_REVEAL_SLOT_REWARD_MULTIPLE: 2 +# 2**8 (= 256) +MINOR_REWARD_QUOTIENT: 256 +# Mainnet preset - Phase0 + +# Misc +# --------------------------------------------------------------- +# 2**6 (= 64) +MAX_COMMITTEES_PER_SLOT: 64 +# 2**7 (= 128) +TARGET_COMMITTEE_SIZE: 128 +# 2**11 (= 2,048) +MAX_VALIDATORS_PER_COMMITTEE: 2048 +# See issue 563 +SHUFFLE_ROUND_COUNT: 90 +# 4 +HYSTERESIS_QUOTIENT: 4 +# 1 (minus 0.25) +HYSTERESIS_DOWNWARD_MULTIPLIER: 1 +# 5 (plus 1.25) +HYSTERESIS_UPWARD_MULTIPLIER: 5 + + +# Fork Choice +# --------------------------------------------------------------- +# 2**3 (= 8) +SAFE_SLOTS_TO_UPDATE_JUSTIFIED: 8 + + +# Gwei values +# --------------------------------------------------------------- +# 2**0 * 10**9 (= 1,000,000,000) Gwei +MIN_DEPOSIT_AMOUNT: 1000000000 +# 2**5 * 10**9 (= 32,000,000,000) Gwei +MAX_EFFECTIVE_BALANCE: 32000000000 +# 2**0 * 10**9 (= 1,000,000,000) Gwei +EFFECTIVE_BALANCE_INCREMENT: 1000000000 + + +# Time parameters +# --------------------------------------------------------------- +# 2**0 (= 1) slots 12 seconds +MIN_ATTESTATION_INCLUSION_DELAY: 1 +# 2**5 (= 32) slots 6.4 minutes +SLOTS_PER_EPOCH: 32 +# 2**0 (= 1) epochs 6.4 minutes +MIN_SEED_LOOKAHEAD: 1 +# 2**2 (= 4) epochs 25.6 minutes +MAX_SEED_LOOKAHEAD: 4 +# 2**6 (= 64) epochs ~6.8 hours +EPOCHS_PER_ETH1_VOTING_PERIOD: 64 +# 2**13 (= 8,192) slots ~27 hours +SLOTS_PER_HISTORICAL_ROOT: 8192 +# 2**2 (= 4) epochs 25.6 minutes +MIN_EPOCHS_TO_INACTIVITY_PENALTY: 4 + + +# State list lengths +# --------------------------------------------------------------- +# 2**16 (= 65,536) epochs ~0.8 years +EPOCHS_PER_HISTORICAL_VECTOR: 65536 +# 2**13 (= 8,192) epochs ~36 days +EPOCHS_PER_SLASHINGS_VECTOR: 8192 +# 2**24 (= 16,777,216) historical roots, ~26,131 years +HISTORICAL_ROOTS_LIMIT: 16777216 +# 2**40 (= 1,099,511,627,776) validator spots +VALIDATOR_REGISTRY_LIMIT: 1099511627776 + + +# Reward and penalty quotients +# --------------------------------------------------------------- +# 2**6 (= 64) +BASE_REWARD_FACTOR: 64 +# 2**9 (= 512) +WHISTLEBLOWER_REWARD_QUOTIENT: 512 +# 2**3 (= 8) +PROPOSER_REWARD_QUOTIENT: 8 +# 2**26 (= 67,108,864) +INACTIVITY_PENALTY_QUOTIENT: 67108864 +# 2**7 (= 128) (lower safety margin at Phase 0 genesis) +MIN_SLASHING_PENALTY_QUOTIENT: 128 +# 1 (lower safety margin at Phase 0 genesis) +PROPORTIONAL_SLASHING_MULTIPLIER: 1 + + +# Max operations per block +# --------------------------------------------------------------- +# 2**4 (= 16) +MAX_PROPOSER_SLASHINGS: 16 +# 2**1 (= 2) +MAX_ATTESTER_SLASHINGS: 2 +# 2**7 (= 128) +MAX_ATTESTATIONS: 128 +# 2**4 (= 16) +MAX_DEPOSITS: 16 +# 2**4 (= 16) +MAX_VOLUNTARY_EXITS: 16 +# Mainnet preset - Sharding + +# Misc +# --------------------------------------------------------------- +# 2**10 (= 1,024) +MAX_SHARDS: 1024 +# 2**6 (= 64) +INITIAL_ACTIVE_SHARDS: 64 +# 2**3 (= 8) +SAMPLE_PRICE_ADJUSTMENT_COEFFICIENT: 8 +# 2**4 (= 16) +MAX_SHARD_PROPOSER_SLASHINGS: 16 +# +MAX_SHARD_HEADERS_PER_SHARD: 4 +# 2**8 (= 256) +SHARD_STATE_MEMORY_SLOTS: 256 +# 2**40 (= 1,099,511,627,776) +BLOB_BUILDER_REGISTRY_LIMIT: 1099511627776 + +# Shard blob samples +# --------------------------------------------------------------- +# 2**11 (= 2,048) +MAX_SAMPLES_PER_BLOCK: 2048 +# 2**10 (= 1,1024) +TARGET_SAMPLES_PER_BLOCK: 1024 + +# Gwei values +# --------------------------------------------------------------- +# 2**33 (= 8,589,934,592) Gwei +MAX_SAMPLE_PRICE: 8589934592 +# 2**3 (= 8) Gwei +MIN_SAMPLE_PRICE: 8 + +# Max operations per block +# --------------------------------------------------------------- +# 2**4 (= 16) +MAX_BLS_TO_EXECUTION_CHANGES: 16 + + +# Execution +# --------------------------------------------------------------- +# [customized] 2**2 (= 4) +MAX_WITHDRAWALS_PER_PAYLOAD: 16 + +# Withdrawals processing +# --------------------------------------------------------------- +# [customized] 2**4 (= 16) validators +MAX_VALIDATORS_PER_WITHDRAWALS_SWEEP: 16384 \ No newline at end of file diff --git a/config/sepolia.chain.yml b/config/sepolia.chain.yml index 6ea256dd18..ceee1752f9 100644 --- a/config/sepolia.chain.yml +++ b/config/sepolia.chain.yml @@ -1,5 +1,3 @@ -# https://github.com/eth-clients/sepolia/blob/0eae813/bepolia/config.yaml - # Extends the mainnet preset PRESET_BASE: 'mainnet' CONFIG_NAME: 'sepolia' @@ -76,3 +74,229 @@ PROPOSER_SCORE_BOOST: 40 DEPOSIT_CHAIN_ID: 11155111 DEPOSIT_NETWORK_ID: 11155111 DEPOSIT_CONTRACT_ADDRESS: 0x7f02C3E3c98b133055B8B348B2Ac625669Ed295D + +# Mainnet preset - Altair + +# Updated penalty values +# --------------------------------------------------------------- +# 3 * 2**24 (= 50,331,648) +INACTIVITY_PENALTY_QUOTIENT_ALTAIR: 50331648 +# 2**6 (= 64) +MIN_SLASHING_PENALTY_QUOTIENT_ALTAIR: 64 +# 2 +PROPORTIONAL_SLASHING_MULTIPLIER_ALTAIR: 2 + + +# Sync committee +# --------------------------------------------------------------- +# 2**9 (= 512) +SYNC_COMMITTEE_SIZE: 512 +# 2**8 (= 256) +EPOCHS_PER_SYNC_COMMITTEE_PERIOD: 256 + + +# Sync protocol +# --------------------------------------------------------------- +# 1 +MIN_SYNC_COMMITTEE_PARTICIPANTS: 1 +# SLOTS_PER_EPOCH * EPOCHS_PER_SYNC_COMMITTEE_PERIOD (= 32 * 256) +UPDATE_TIMEOUT: 8192 +# Mainnet preset - Bellatrix + +# Updated penalty values +# --------------------------------------------------------------- +# 2**24 (= 16,777,216) +INACTIVITY_PENALTY_QUOTIENT_BELLATRIX: 16777216 +# 2**5 (= 32) +MIN_SLASHING_PENALTY_QUOTIENT_BELLATRIX: 32 +# 3 +PROPORTIONAL_SLASHING_MULTIPLIER_BELLATRIX: 3 + +# Execution +# --------------------------------------------------------------- +# 2**30 (= 1,073,741,824) +MAX_BYTES_PER_TRANSACTION: 1073741824 +# 2**20 (= 1,048,576) +MAX_TRANSACTIONS_PER_PAYLOAD: 1048576 +# 2**8 (= 256) +BYTES_PER_LOGS_BLOOM: 256 +# 2**5 (= 32) +MAX_EXTRA_DATA_BYTES: 32 +# Minimal preset - Capella +# Mainnet preset - Custody Game + +# Time parameters +# --------------------------------------------------------------- +# 2**1 (= 2) epochs, 12.8 minutes +RANDAO_PENALTY_EPOCHS: 2 +# 2**15 (= 32,768) epochs, ~146 days +EARLY_DERIVED_SECRET_PENALTY_MAX_FUTURE_EPOCHS: 32768 +# 2**14 (= 16,384) epochs ~73 days +EPOCHS_PER_CUSTODY_PERIOD: 16384 +# 2**11 (= 2,048) epochs, ~9 days +CUSTODY_PERIOD_TO_RANDAO_PADDING: 2048 +# 2**15 (= 32,768) epochs, ~146 days +MAX_CHUNK_CHALLENGE_DELAY: 32768 + + +# Max operations +# --------------------------------------------------------------- +# 2**8 (= 256) +MAX_CUSTODY_KEY_REVEALS: 256 +# 2**0 (= 1) +MAX_EARLY_DERIVED_SECRET_REVEALS: 1 +# 2**2 (= 2) +MAX_CUSTODY_CHUNK_CHALLENGES: 4 +# 2** 4 (= 16) +MAX_CUSTODY_CHUNK_CHALLENGE_RESP: 16 +# 2**0 (= 1) +MAX_CUSTODY_SLASHINGS: 1 + + +# Reward and penalty quotients +# --------------------------------------------------------------- +EARLY_DERIVED_SECRET_REVEAL_SLOT_REWARD_MULTIPLE: 2 +# 2**8 (= 256) +MINOR_REWARD_QUOTIENT: 256 +# Mainnet preset - Phase0 + +# Misc +# --------------------------------------------------------------- +# 2**6 (= 64) +MAX_COMMITTEES_PER_SLOT: 64 +# 2**7 (= 128) +TARGET_COMMITTEE_SIZE: 128 +# 2**11 (= 2,048) +MAX_VALIDATORS_PER_COMMITTEE: 2048 +# See issue 563 +SHUFFLE_ROUND_COUNT: 90 +# 4 +HYSTERESIS_QUOTIENT: 4 +# 1 (minus 0.25) +HYSTERESIS_DOWNWARD_MULTIPLIER: 1 +# 5 (plus 1.25) +HYSTERESIS_UPWARD_MULTIPLIER: 5 + + +# Fork Choice +# --------------------------------------------------------------- +# 2**3 (= 8) +SAFE_SLOTS_TO_UPDATE_JUSTIFIED: 8 + + +# Gwei values +# --------------------------------------------------------------- +# 2**0 * 10**9 (= 1,000,000,000) Gwei +MIN_DEPOSIT_AMOUNT: 1000000000 +# 2**5 * 10**9 (= 32,000,000,000) Gwei +MAX_EFFECTIVE_BALANCE: 32000000000 +# 2**0 * 10**9 (= 1,000,000,000) Gwei +EFFECTIVE_BALANCE_INCREMENT: 1000000000 + + +# Time parameters +# --------------------------------------------------------------- +# 2**0 (= 1) slots 12 seconds +MIN_ATTESTATION_INCLUSION_DELAY: 1 +# 2**5 (= 32) slots 6.4 minutes +SLOTS_PER_EPOCH: 32 +# 2**0 (= 1) epochs 6.4 minutes +MIN_SEED_LOOKAHEAD: 1 +# 2**2 (= 4) epochs 25.6 minutes +MAX_SEED_LOOKAHEAD: 4 +# 2**6 (= 64) epochs ~6.8 hours +EPOCHS_PER_ETH1_VOTING_PERIOD: 64 +# 2**13 (= 8,192) slots ~27 hours +SLOTS_PER_HISTORICAL_ROOT: 8192 +# 2**2 (= 4) epochs 25.6 minutes +MIN_EPOCHS_TO_INACTIVITY_PENALTY: 4 + + +# State list lengths +# --------------------------------------------------------------- +# 2**16 (= 65,536) epochs ~0.8 years +EPOCHS_PER_HISTORICAL_VECTOR: 65536 +# 2**13 (= 8,192) epochs ~36 days +EPOCHS_PER_SLASHINGS_VECTOR: 8192 +# 2**24 (= 16,777,216) historical roots, ~26,131 years +HISTORICAL_ROOTS_LIMIT: 16777216 +# 2**40 (= 1,099,511,627,776) validator spots +VALIDATOR_REGISTRY_LIMIT: 1099511627776 + + +# Reward and penalty quotients +# --------------------------------------------------------------- +# 2**6 (= 64) +BASE_REWARD_FACTOR: 64 +# 2**9 (= 512) +WHISTLEBLOWER_REWARD_QUOTIENT: 512 +# 2**3 (= 8) +PROPOSER_REWARD_QUOTIENT: 8 +# 2**26 (= 67,108,864) +INACTIVITY_PENALTY_QUOTIENT: 67108864 +# 2**7 (= 128) (lower safety margin at Phase 0 genesis) +MIN_SLASHING_PENALTY_QUOTIENT: 128 +# 1 (lower safety margin at Phase 0 genesis) +PROPORTIONAL_SLASHING_MULTIPLIER: 1 + + +# Max operations per block +# --------------------------------------------------------------- +# 2**4 (= 16) +MAX_PROPOSER_SLASHINGS: 16 +# 2**1 (= 2) +MAX_ATTESTER_SLASHINGS: 2 +# 2**7 (= 128) +MAX_ATTESTATIONS: 128 +# 2**4 (= 16) +MAX_DEPOSITS: 16 +# 2**4 (= 16) +MAX_VOLUNTARY_EXITS: 16 +# Mainnet preset - Sharding + +# Misc +# --------------------------------------------------------------- +# 2**10 (= 1,024) +MAX_SHARDS: 1024 +# 2**6 (= 64) +INITIAL_ACTIVE_SHARDS: 64 +# 2**3 (= 8) +SAMPLE_PRICE_ADJUSTMENT_COEFFICIENT: 8 +# 2**4 (= 16) +MAX_SHARD_PROPOSER_SLASHINGS: 16 +# +MAX_SHARD_HEADERS_PER_SHARD: 4 +# 2**8 (= 256) +SHARD_STATE_MEMORY_SLOTS: 256 +# 2**40 (= 1,099,511,627,776) +BLOB_BUILDER_REGISTRY_LIMIT: 1099511627776 + +# Shard blob samples +# --------------------------------------------------------------- +# 2**11 (= 2,048) +MAX_SAMPLES_PER_BLOCK: 2048 +# 2**10 (= 1,1024) +TARGET_SAMPLES_PER_BLOCK: 1024 + +# Gwei values +# --------------------------------------------------------------- +# 2**33 (= 8,589,934,592) Gwei +MAX_SAMPLE_PRICE: 8589934592 +# 2**3 (= 8) Gwei +MIN_SAMPLE_PRICE: 8 + +# Max operations per block +# --------------------------------------------------------------- +# 2**4 (= 16) +MAX_BLS_TO_EXECUTION_CHANGES: 16 + + +# Execution +# --------------------------------------------------------------- +# [customized] 2**2 (= 4) +MAX_WITHDRAWALS_PER_PAYLOAD: 16 + +# Withdrawals processing +# --------------------------------------------------------------- +# [customized] 2**4 (= 16) validators +MAX_VALIDATORS_PER_WITHDRAWALS_SWEEP: 16384 \ No newline at end of file diff --git a/config/testnet.chain.yml b/config/testnet.chain.yml new file mode 100644 index 0000000000..8a27f1208c --- /dev/null +++ b/config/testnet.chain.yml @@ -0,0 +1,129 @@ +# Extends the mainnet preset +PRESET_BASE: 'mainnet' +CONFIG_NAME: testnet # needs to exist because of Prysm. Otherwise it conflicts with mainnet genesis + +# Genesis +# --------------------------------------------------------------- +# `2**14` (= 16,384) +MIN_GENESIS_ACTIVE_VALIDATOR_COUNT: 58000 +# Feb-01-2023 15:00:00 PM UTC +# This is an invalid valid and should be updated when you create the genesis +MIN_GENESIS_TIME: 1675263480 +GENESIS_FORK_VERSION: 0x00000069 +GENESIS_DELAY: 120 + + +# Forking +# --------------------------------------------------------------- +# Some forks are disabled for now: +# - These may be re-assigned to another fork-version later +# - Temporarily set to max uint64 value: 2**64 - 1 + +# Altair +ALTAIR_FORK_VERSION: 0x00000070 +ALTAIR_FORK_EPOCH: 0 +# Merge +BELLATRIX_FORK_VERSION: 0x00000071 +BELLATRIX_FORK_EPOCH: 0 +TERMINAL_TOTAL_DIFFICULTY: 0 +TERMINAL_BLOCK_HASH: 0x0000000000000000000000000000000000000000000000000000000000000000 +TERMINAL_BLOCK_HASH_ACTIVATION_EPOCH: 18446744073709551615 + +# Capella +CAPELLA_FORK_VERSION: 0x00000072 +CAPELLA_FORK_EPOCH: 1350 + +# EIP4844 +EIP4844_FORK_VERSION: 0x00000073 +EIP4844_FORK_EPOCH: 18446744073709551615 + +# Time parameters +# --------------------------------------------------------------- +# 12 seconds +SECONDS_PER_SLOT: 12 +# 14 (estimate from Eth1 mainnet) +SECONDS_PER_ETH1_BLOCK: 14 +# 2**8 (= 256) epochs ~27 hours +MIN_VALIDATOR_WITHDRAWABILITY_DELAY: 256 +# 2**8 (= 256) epochs ~27 hours +SHARD_COMMITTEE_PERIOD: 256 +# 2**11 (= 2,048) Eth1 blocks ~8 hours +ETH1_FOLLOW_DISTANCE: 12 + + +# Validator cycle +# --------------------------------------------------------------- +# 2**2 (= 4) +INACTIVITY_SCORE_BIAS: 4 +# 2**4 (= 16) +INACTIVITY_SCORE_RECOVERY_RATE: 16 +# 2**4 * 10**9 (= 16,000,000,000) Gwei +EJECTION_BALANCE: 16000000000 +# 2**2 (= 4) +MIN_PER_EPOCH_CHURN_LIMIT: 4 +# 2**16 (= 65,536) +CHURN_LIMIT_QUOTIENT: 65536 + +# Fork choice +# --------------------------------------------------------------- +# 40% +PROPOSER_SCORE_BOOST: 40 + +# Deposit contract +# --------------------------------------------------------------- +DEPOSIT_CHAIN_ID: 1337803 +DEPOSIT_NETWORK_ID: 1337803 +DEPOSIT_CONTRACT_ADDRESS: 0x4242424242424242424242424242424242424242 +INACTIVITY_PENALTY_QUOTIENT_ALTAIR: 50331648 +MIN_SLASHING_PENALTY_QUOTIENT_ALTAIR: 64 +PROPORTIONAL_SLASHING_MULTIPLIER_ALTAIR: 2 +SYNC_COMMITTEE_SIZE: 512 +EPOCHS_PER_SYNC_COMMITTEE_PERIOD: 256 +MIN_SYNC_COMMITTEE_PARTICIPANTS: 1 +UPDATE_TIMEOUT: 8192 +INACTIVITY_PENALTY_QUOTIENT_BELLATRIX: 16777216 +MIN_SLASHING_PENALTY_QUOTIENT_BELLATRIX: 32 +PROPORTIONAL_SLASHING_MULTIPLIER_BELLATRIX: 3 +MAX_BYTES_PER_TRANSACTION: 1073741824 +MAX_TRANSACTIONS_PER_PAYLOAD: 1048576 +BYTES_PER_LOGS_BLOOM: 256 +MAX_EXTRA_DATA_BYTES: 32 +RANDAO_PENALTY_EPOCHS: 2 +EARLY_DERIVED_SECRET_PENALTY_MAX_FUTURE_EPOCHS: 32768 +EPOCHS_PER_CUSTODY_PERIOD: 16384 +CUSTODY_PERIOD_TO_RANDAO_PADDING: 2048 +MAX_CHUNK_CHALLENGE_DELAY: 32768 +MAX_CUSTODY_KEY_REVEALS: 256 +MAX_EARLY_DERIVED_SECRET_REVEALS: 1 +MAX_CUSTODY_CHUNK_CHALLENGES: 4 +MAX_CUSTODY_CHUNK_CHALLENGE_RESP: 16 +MAX_CUSTODY_SLASHINGS: 1 +EARLY_DERIVED_SECRET_REVEAL_SLOT_REWARD_MULTIPLE: 2 +MINOR_REWARD_QUOTIENT: 256 +MAX_COMMITTEES_PER_SLOT: 64 +TARGET_COMMITTEE_SIZE: 128 +MAX_VALIDATORS_PER_COMMITTEE: 2048 +SHUFFLE_ROUND_COUNT: 90 +HYSTERESIS_QUOTIENT: 4 +SLOTS_PER_EPOCH: 32 +MAX_EFFECTIVE_BALANCE: 32000000000 + + + +# Minimal preset - Capella + +# Max operations per block +# --------------------------------------------------------------- +# 2**4 (= 16) +MAX_BLS_TO_EXECUTION_CHANGES: 16 + + +# Execution +# --------------------------------------------------------------- +# [customized] 2**2 (= 4) +MAX_WITHDRAWALS_PER_PAYLOAD: 4 + +# Withdrawals processing +# --------------------------------------------------------------- +# [customized] 2**4 (= 16) validators +MAX_VALIDATORS_PER_WITHDRAWALS_SWEEP: 16 \ No newline at end of file diff --git a/db/bigtable.go b/db/bigtable.go index 99ef341131..fe69cb60e9 100644 --- a/db/bigtable.go +++ b/db/bigtable.go @@ -6,6 +6,8 @@ import ( "eth2-exporter/types" "eth2-exporter/utils" "fmt" + "math" + "os" "sort" "strconv" "strings" @@ -15,7 +17,7 @@ import ( gcp_bigtable "cloud.google.com/go/bigtable" "github.com/go-redis/redis/v8" itypes "github.com/gobitfly/eth-rewards/types" - utilMath "github.com/protolambda/zrnt/eth2/util/math" + "github.com/sirupsen/logrus" "golang.org/x/sync/errgroup" "google.golang.org/api/option" "google.golang.org/protobuf/proto" @@ -24,33 +26,42 @@ import ( var BigtableClient *Bigtable const ( - DEFAULT_FAMILY = "f" - VALIDATOR_BALANCES_FAMILY = "vb" - ATTESTATIONS_FAMILY = "at" - PROPOSALS_FAMILY = "pr" - SYNC_COMMITTEES_FAMILY = "sc" - INCOME_DETAILS_COLUMN_FAMILY = "id" - STATS_COLUMN_FAMILY = "stats" - MACHINE_METRICS_COLUMN_FAMILY = "mm" - SERIES_FAMILY = "series" + DEFAULT_FAMILY = "f" + VALIDATOR_BALANCES_FAMILY = "vb" + VALIDATOR_HIGHEST_ACTIVE_INDEX_FAMILY = "ha" + ATTESTATIONS_FAMILY = "at" + PROPOSALS_FAMILY = "pr" + SYNC_COMMITTEES_FAMILY = "sc" + SYNC_COMMITTEES_PARTICIPATION_FAMILY = "sp" + INCOME_DETAILS_COLUMN_FAMILY = "id" + STATS_COLUMN_FAMILY = "stats" + MACHINE_METRICS_COLUMN_FAMILY = "mm" + SERIES_FAMILY = "series" SUM_COLUMN = "sum" - max_block_number = 1000000000 - max_epoch = 1000000000 + MAX_CL_BLOCK_NUMBER = 1000000000 - 1 + MAX_EL_BLOCK_NUMBER = 1000000000 + MAX_EPOCH = 1000000000 - 1 + + MAX_BATCH_MUTATIONS = 100000 + + REPORT_TIMEOUT = time.Second * 10 ) type Bigtable struct { client *gcp_bigtable.Client - tableBeaconchain *gcp_bigtable.Table + tableBeaconchain *gcp_bigtable.Table + tableValidators *gcp_bigtable.Table + tableValidatorsHistory *gcp_bigtable.Table tableData *gcp_bigtable.Table tableBlocks *gcp_bigtable.Table tableMetadataUpdates *gcp_bigtable.Table tableMetadata *gcp_bigtable.Table - tableMachineMetrics *gcp_bigtable.Table - tableValidators *gcp_bigtable.Table + + tableMachineMetrics *gcp_bigtable.Table redisCache *redis.Client @@ -61,6 +72,19 @@ type Bigtable struct { } func InitBigtable(project, instance, chainId, redisAddress string) (*Bigtable, error) { + + if utils.Config.Bigtable.Emulator { + + if utils.Config.Bigtable.EmulatorHost == "" { + utils.Config.Bigtable.EmulatorHost = "127.0.0.1" + } + logger.Infof("using emulated local bigtable environment, setting BIGTABLE_EMULATOR_HOST env variable to %s:%d", utils.Config.Bigtable.EmulatorHost, utils.Config.Bigtable.EmulatorPort) + err := os.Setenv("BIGTABLE_EMULATOR_HOST", fmt.Sprintf("%s:%d", utils.Config.Bigtable.EmulatorHost, utils.Config.Bigtable.EmulatorPort)) + + if err != nil { + logger.Fatalf("unable to set bigtable emulator environment variable: %v", err) + } + } ctx, cancel := context.WithTimeout(context.Background(), time.Second*30) defer cancel() @@ -90,6 +114,7 @@ func InitBigtable(project, instance, chainId, redisAddress string) (*Bigtable, e tableBeaconchain: btClient.Open("beaconchain"), tableMachineMetrics: btClient.Open("machine_metrics"), tableValidators: btClient.Open("beaconchain_validators"), + tableValidatorsHistory: btClient.Open("beaconchain_validators_history"), chainId: chainId, redisCache: rdc, lastAttestationCacheMux: &sync.Mutex{}, @@ -111,7 +136,7 @@ func (bigtable *Bigtable) SaveMachineMetric(process string, userID uint64, machi ctx, cancel := context.WithTimeout(context.Background(), time.Second*30) defer cancel() - rowKeyData := fmt.Sprintf("u:%s:p:%s:m:%v", reversePaddedUserID(userID), process, machine) + rowKeyData := fmt.Sprintf("u:%s:p:%s:m:%v", bigtable.reversePaddedUserID(userID), process, machine) ts := gcp_bigtable.Now() rateLimitKey := fmt.Sprintf("%s:%d", rowKeyData, ts.Time().Minute()) @@ -125,7 +150,7 @@ func (bigtable *Bigtable) SaveMachineMetric(process string, userID uint64, machi // for limiting machines per user, add the machine field to a redis set // bucket period is 15mins - machineLimitKey := fmt.Sprintf("%s:%d", reversePaddedUserID(userID), ts.Time().Minute()%15) + machineLimitKey := fmt.Sprintf("%s:%d", bigtable.reversePaddedUserID(userID), ts.Time().Minute()%15) pipe := bigtable.redisCache.Pipeline() pipe.SAdd(ctx, machineLimitKey, machine) pipe.Expire(ctx, machineLimitKey, time.Minute*15) @@ -153,7 +178,7 @@ func (bigtable Bigtable) getMachineMetricNamesMap(userID uint64, searchDepth int ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(time.Second*30)) defer cancel() - rangePrefix := fmt.Sprintf("u:%s:p:", reversePaddedUserID(userID)) + rangePrefix := fmt.Sprintf("u:%s:p:", bigtable.reversePaddedUserID(userID)) filter := gcp_bigtable.ChainFilters( gcp_bigtable.FamilyFilter(MACHINE_METRICS_COLUMN_FAMILY), @@ -181,6 +206,15 @@ func (bigtable Bigtable) getMachineMetricNamesMap(userID uint64, searchDepth int } func (bigtable Bigtable) GetMachineMetricsMachineNames(userID uint64) ([]string, error) { + tmr := time.NewTimer(REPORT_TIMEOUT) + defer tmr.Stop() + go func() { + <-tmr.C + logger.WithFields(logrus.Fields{ + "userId": userID, + }).Warnf("%s call took longer than %v", utils.GetCurrentFuncName(), REPORT_TIMEOUT) + }() + names, err := bigtable.getMachineMetricNamesMap(userID, 300) if err != nil { return nil, err @@ -195,10 +229,19 @@ func (bigtable Bigtable) GetMachineMetricsMachineNames(userID uint64) ([]string, } func (bigtable Bigtable) GetMachineMetricsMachineCount(userID uint64) (uint64, error) { + tmr := time.NewTimer(REPORT_TIMEOUT) + defer tmr.Stop() + go func() { + <-tmr.C + logger.WithFields(logrus.Fields{ + "userId": userID, + }).Warnf("%s call took longer than %v", utils.GetCurrentFuncName(), REPORT_TIMEOUT) + }() + ctx, cancel := context.WithTimeout(context.Background(), time.Second*10) defer cancel() - machineLimitKey := fmt.Sprintf("%s:%d", reversePaddedUserID(userID), time.Now().Minute()%15) + machineLimitKey := fmt.Sprintf("%s:%d", bigtable.reversePaddedUserID(userID), time.Now().Minute()%15) card, err := bigtable.redisCache.SCard(ctx, machineLimitKey).Result() if err != nil { @@ -208,6 +251,17 @@ func (bigtable Bigtable) GetMachineMetricsMachineCount(userID uint64) (uint64, e } func (bigtable Bigtable) GetMachineMetricsNode(userID uint64, limit, offset int) ([]*types.MachineMetricNode, error) { + tmr := time.NewTimer(REPORT_TIMEOUT) + defer tmr.Stop() + go func() { + <-tmr.C + logger.WithFields(logrus.Fields{ + "userId": userID, + "limit": limit, + "offset": offset, + }).Warnf("%s call took longer than %v", utils.GetCurrentFuncName(), REPORT_TIMEOUT) + }() + return getMachineMetrics(bigtable, "beaconnode", userID, limit, offset, func(data []byte, machine string) *types.MachineMetricNode { obj := &types.MachineMetricNode{} @@ -222,6 +276,17 @@ func (bigtable Bigtable) GetMachineMetricsNode(userID uint64, limit, offset int) } func (bigtable Bigtable) GetMachineMetricsValidator(userID uint64, limit, offset int) ([]*types.MachineMetricValidator, error) { + tmr := time.NewTimer(REPORT_TIMEOUT) + defer tmr.Stop() + go func() { + <-tmr.C + logger.WithFields(logrus.Fields{ + "userId": userID, + "limit": limit, + "offset": offset, + }).Warnf("%s call took longer than %v", utils.GetCurrentFuncName(), REPORT_TIMEOUT) + }() + return getMachineMetrics(bigtable, "validator", userID, limit, offset, func(data []byte, machine string) *types.MachineMetricValidator { obj := &types.MachineMetricValidator{} @@ -236,6 +301,17 @@ func (bigtable Bigtable) GetMachineMetricsValidator(userID uint64, limit, offset } func (bigtable Bigtable) GetMachineMetricsSystem(userID uint64, limit, offset int) ([]*types.MachineMetricSystem, error) { + tmr := time.NewTimer(REPORT_TIMEOUT) + defer tmr.Stop() + go func() { + <-tmr.C + logger.WithFields(logrus.Fields{ + "userId": userID, + "limit": limit, + "offset": offset, + }).Warnf("%s call took longer than %v", utils.GetCurrentFuncName(), REPORT_TIMEOUT) + }() + return getMachineMetrics(bigtable, "system", userID, limit, offset, func(data []byte, machine string) *types.MachineMetricSystem { obj := &types.MachineMetricSystem{} @@ -253,7 +329,7 @@ func getMachineMetrics[T types.MachineMetricSystem | types.MachineMetricNode | t ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(time.Second*30)) defer cancel() - rangePrefix := fmt.Sprintf("u:%s:p:%s:m:", reversePaddedUserID(userID), process) + rangePrefix := fmt.Sprintf("u:%s:p:%s:m:", bigtable.reversePaddedUserID(userID), process) res := make([]*T, 0) if offset <= 0 { offset = 1 @@ -293,8 +369,8 @@ func getMachineMetrics[T types.MachineMetricSystem | types.MachineMetricNode | t return res, nil } -func GetMachineRowKey(userID uint64, process string, machine string) string { - return fmt.Sprintf("u:%s:p:%s:m:%s", reversePaddedUserID(userID), process, machine) +func (bigtable Bigtable) GetMachineRowKey(userID uint64, process string, machine string) string { + return fmt.Sprintf("u:%s:p:%s:m:%s", bigtable.reversePaddedUserID(userID), process, machine) } // Returns a map[userID]map[machineName]machineData @@ -302,6 +378,15 @@ func GetMachineRowKey(userID uint64, process string, machine string) string { // and 5 minute old data in fiveMinuteOldData (defined in limit) // as well as the insert timestamps of both func (bigtable Bigtable) GetMachineMetricsForNotifications(rowKeys gcp_bigtable.RowList) (map[uint64]map[string]*types.MachineMetricSystemUser, error) { + tmr := time.NewTimer(REPORT_TIMEOUT) + defer tmr.Stop() + go func() { + <-tmr.C + logger.WithFields(logrus.Fields{ + "rowKeys": rowKeys, + }).Warnf("%s call took longer than %v", utils.GetCurrentFuncName(), REPORT_TIMEOUT) + }() + ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(time.Second*200)) defer cancel() @@ -388,15 +473,23 @@ func machineMetricRowParts(r string) (bool, uint64, string, string) { func (bigtable *Bigtable) SaveValidatorBalances(epoch uint64, validators []*types.Validator) error { - ctx, cancel := context.WithTimeout(context.Background(), time.Second*30) + ctx, cancel := context.WithTimeout(context.Background(), time.Minute*2) defer cancel() - start := time.Now() + // start := time.Now() ts := gcp_bigtable.Timestamp(0) - mut := gcp_bigtable.NewMutation() + muts := make([]*gcp_bigtable.Mutation, 0, MAX_BATCH_MUTATIONS) + keys := make([]string, 0, MAX_BATCH_MUTATIONS) + + highestActiveIndex := uint64(0) + epochKey := bigtable.reversedPaddedEpoch(epoch) + for _, validator := range validators { + + if validator.Balance > 0 && validator.Index > highestActiveIndex { + highestActiveIndex = validator.Index + } - for i, validator := range validators { balanceEncoded := make([]byte, 8) binary.LittleEndian.PutUint64(balanceEncoded, validator.Balance) @@ -404,63 +497,53 @@ func (bigtable *Bigtable) SaveValidatorBalances(epoch uint64, validators []*type binary.LittleEndian.PutUint64(effectiveBalanceEncoded, validator.EffectiveBalance) combined := append(balanceEncoded, effectiveBalanceEncoded...) - mut.Set(VALIDATOR_BALANCES_FAMILY, fmt.Sprintf("%d", validator.Index), ts, combined) + mut := &gcp_bigtable.Mutation{} + mut.Set(VALIDATOR_BALANCES_FAMILY, "b", ts, combined) + key := fmt.Sprintf("%s:%s:%s:%s", bigtable.chainId, bigtable.validatorIndexToKey(validator.Index), VALIDATOR_BALANCES_FAMILY, epochKey) + + muts = append(muts, mut) + keys = append(keys, key) - if i%100000 == 0 { - err := bigtable.tableBeaconchain.Apply(ctx, fmt.Sprintf("%s:e:b:%s", bigtable.chainId, reversedPaddedEpoch(epoch)), mut) + if len(muts) == MAX_BATCH_MUTATIONS { + errs, err := bigtable.tableValidatorsHistory.ApplyBulk(ctx, keys, muts) if err != nil { return err } - mut = gcp_bigtable.NewMutation() - } - } - err := bigtable.tableBeaconchain.Apply(ctx, fmt.Sprintf("%s:e:b:%s", bigtable.chainId, reversedPaddedEpoch(epoch)), mut) - if err != nil { - return err + for _, err := range errs { + return err + } + muts = make([]*gcp_bigtable.Mutation, 0, MAX_BATCH_MUTATIONS) + keys = make([]string, 0, MAX_BATCH_MUTATIONS) + } } - logger.Infof("exported validator balances to bigtable in %v", time.Since(start)) - return nil -} - -func (bigtable *Bigtable) SaveAttestationAssignments(epoch uint64, assignments map[string]uint64) error { - - ctx, cancel := context.WithTimeout(context.Background(), time.Second*30) - defer cancel() - - start := time.Now() - ts := gcp_bigtable.Timestamp(0) - - validatorsPerSlot := make(map[uint64][]uint64) - for key, validator := range assignments { - keySplit := strings.Split(key, "-") + if len(muts) > 0 { + errs, err := bigtable.tableValidatorsHistory.ApplyBulk(ctx, keys, muts) - attesterslot, err := strconv.ParseUint(keySplit[0], 10, 64) if err != nil { return err } - if validatorsPerSlot[attesterslot] == nil { - validatorsPerSlot[attesterslot] = make([]uint64, 0, len(assignments)/int(utils.Config.Chain.ClConfig.SlotsPerEpoch)) + for _, err := range errs { + return err } - validatorsPerSlot[attesterslot] = append(validatorsPerSlot[attesterslot], validator) } - for slot, validators := range validatorsPerSlot { - mut := gcp_bigtable.NewMutation() - for _, validator := range validators { - mut.Set(ATTESTATIONS_FAMILY, fmt.Sprintf("%d", validator), ts, []byte{}) - } - err := bigtable.tableBeaconchain.Apply(ctx, fmt.Sprintf("%s:e:%s:s:%s", bigtable.chainId, reversedPaddedEpoch(epoch), reversedPaddedSlot(slot)), mut) + // store the highes active validator index for that epoch + highestActiveIndexEncoded := make([]byte, 8) + binary.LittleEndian.PutUint64(highestActiveIndexEncoded, highestActiveIndex) - if err != nil { - return err - } + mut := &gcp_bigtable.Mutation{} + mut.Set(VALIDATOR_HIGHEST_ACTIVE_INDEX_FAMILY, VALIDATOR_HIGHEST_ACTIVE_INDEX_FAMILY, ts, highestActiveIndexEncoded) + key := fmt.Sprintf("%s:%s:%s", bigtable.chainId, VALIDATOR_HIGHEST_ACTIVE_INDEX_FAMILY, epochKey) + err := bigtable.tableValidatorsHistory.Apply(ctx, key, mut) + if err != nil { + return err } - logger.Infof("exported attestation assignments to bigtable in %v", time.Since(start)) + // logger.Infof("exported validator balances to bigtable in %v", time.Since(start)) return nil } @@ -472,60 +555,50 @@ func (bigtable *Bigtable) SaveProposalAssignments(epoch uint64, assignments map[ start := time.Now() ts := gcp_bigtable.Timestamp(0) + muts := make([]*gcp_bigtable.Mutation, 0, len(assignments)) + keys := make([]string, 0, len(assignments)) + for slot, validator := range assignments { mut := gcp_bigtable.NewMutation() - mut.Set(PROPOSALS_FAMILY, fmt.Sprintf("%d", validator), ts, []byte{}) - err := bigtable.tableBeaconchain.Apply(ctx, fmt.Sprintf("%s:e:%s:s:%s", bigtable.chainId, reversedPaddedEpoch(epoch), reversedPaddedSlot(slot)), mut) - - if err != nil { - return err - } - } + mut.Set(PROPOSALS_FAMILY, "p", ts, []byte{}) - logger.Infof("exported proposal assignments to bigtable in %v", time.Since(start)) - return nil -} - -func (bigtable *Bigtable) SaveSyncCommitteesAssignments(startSlot, endSlot uint64, validators []uint64) error { + key := fmt.Sprintf("%s:%s:%s:%s:%s", bigtable.chainId, bigtable.validatorIndexToKey(validator), PROPOSALS_FAMILY, bigtable.reversedPaddedEpoch(epoch), bigtable.reversedPaddedSlot(slot)) - ctx, cancel := context.WithTimeout(context.Background(), time.Minute*5) - defer cancel() + muts = append(muts, mut) + keys = append(keys, key) - start := time.Now() - ts := gcp_bigtable.Timestamp(0) + if len(muts) == MAX_BATCH_MUTATIONS { + errs, err := bigtable.tableValidatorsHistory.ApplyBulk(ctx, keys, muts) - var muts []*gcp_bigtable.Mutation - var keys []string + if err != nil { + return err + } - for i := startSlot; i <= endSlot; i++ { - mut := gcp_bigtable.NewMutation() - for _, validator := range validators { - mut.Set(SYNC_COMMITTEES_FAMILY, fmt.Sprintf("%d", validator), ts, []byte{}) + for _, err := range errs { + return err + } + muts = make([]*gcp_bigtable.Mutation, 0, MAX_BATCH_MUTATIONS) + keys = make([]string, 0, MAX_BATCH_MUTATIONS) } - - muts = append(muts, mut) - keys = append(keys, fmt.Sprintf("%s:e:%s:s:%s", bigtable.chainId, reversedPaddedEpoch(i/utils.Config.Chain.ClConfig.SlotsPerEpoch), reversedPaddedSlot(i))) } - logger.Infof("saving %v mutations for sync duties", len(muts)) - - errs, err := bigtable.tableBeaconchain.ApplyBulk(ctx, keys, muts) - - if err != nil { - return err - } + if len(muts) > 0 { + errs, err := bigtable.tableValidatorsHistory.ApplyBulk(ctx, keys, muts) - for _, err := range errs { if err != nil { return err } + + for _, err := range errs { + return err + } } - logger.Infof("exported sync committee assignments to bigtable in %v", time.Since(start)) + logger.Infof("exported proposal assignments to bigtable in %v", time.Since(start)) return nil } -func (bigtable *Bigtable) SaveAttestations(blocks map[uint64]map[string]*types.Block) error { +func (bigtable *Bigtable) SaveAttestationDuties(duties map[types.Slot]map[types.ValidatorIndex][]types.Slot) error { // Initialize in memory last attestation cache lazily bigtable.lastAttestationCacheMux.Lock() @@ -543,72 +616,93 @@ func (bigtable *Bigtable) SaveAttestations(blocks map[uint64]map[string]*types.B } bigtable.lastAttestationCacheMux.Unlock() - ctx, cancel := context.WithTimeout(context.Background(), time.Second*30) + ctx, cancel := context.WithTimeout(context.Background(), time.Minute*2) defer cancel() start := time.Now() - attestationsBySlot := make(map[uint64]map[uint64]uint64) //map[attestedSlot]map[validator]includedSlot + mutsInclusionSlot := make([]*gcp_bigtable.Mutation, 0, MAX_BATCH_MUTATIONS) + keysInclusionSlot := make([]string, 0, MAX_BATCH_MUTATIONS) - slots := make([]uint64, 0, len(blocks)) - for slot := range blocks { - slots = append(slots, slot) - } - sort.Slice(slots, func(i, j int) bool { - return slots[i] < slots[j] - }) + writes := 0 - for _, slot := range slots { - for _, b := range blocks[slot] { - logger.Infof("processing slot %v", slot) - for _, a := range b.Attestations { - for _, validator := range a.Attesters { - inclusionSlot := slot - attestedSlot := a.Data.Slot - if attestationsBySlot[attestedSlot] == nil { - attestationsBySlot[attestedSlot] = make(map[uint64]uint64) - } + mutLastAttestationSlot := gcp_bigtable.NewMutation() + mutLastAttestationSlotCount := 0 + + for attestedSlot, validators := range duties { + for validator, inclusions := range validators { - if attestationsBySlot[attestedSlot][validator] == 0 || inclusionSlot < attestationsBySlot[attestedSlot][validator] { - attestationsBySlot[attestedSlot][validator] = inclusionSlot + epoch := utils.EpochOfSlot(uint64(attestedSlot)) + bigtable.lastAttestationCacheMux.Lock() + if len(inclusions) == 0 { // for missed attestations we write the max block number which will yield a cell ts of 0 + inclusions = append(inclusions, MAX_CL_BLOCK_NUMBER) + } + for _, inclusionSlot := range inclusions { + mutInclusionSlot := gcp_bigtable.NewMutation() + mutInclusionSlot.Set(ATTESTATIONS_FAMILY, fmt.Sprintf("%d", attestedSlot), gcp_bigtable.Timestamp((MAX_CL_BLOCK_NUMBER-inclusionSlot)*1000), []byte{}) + key := fmt.Sprintf("%s:%s:%s:%s", bigtable.chainId, bigtable.validatorIndexToKey(uint64(validator)), ATTESTATIONS_FAMILY, bigtable.reversedPaddedEpoch(epoch)) + + mutsInclusionSlot = append(mutsInclusionSlot, mutInclusionSlot) + keysInclusionSlot = append(keysInclusionSlot, key) + writes++ + + if inclusionSlot != MAX_CL_BLOCK_NUMBER && uint64(attestedSlot) > bigtable.lastAttestationCache[uint64(validator)] { + mutLastAttestationSlot.Set(ATTESTATIONS_FAMILY, fmt.Sprintf("%d", validator), gcp_bigtable.Timestamp((attestedSlot)*1000), []byte{}) + bigtable.lastAttestationCache[uint64(validator)] = uint64(attestedSlot) + mutLastAttestationSlotCount++ + + if mutLastAttestationSlotCount == MAX_BATCH_MUTATIONS { + mutStart := time.Now() + err := bigtable.tableValidators.Apply(ctx, fmt.Sprintf("%s:lastAttestationSlot", bigtable.chainId), mutLastAttestationSlot) + if err != nil { + return fmt.Errorf("error applying last attestation slot mutations: %v", err) + } + mutLastAttestationSlot = gcp_bigtable.NewMutation() + mutLastAttestationSlotCount = 0 + logger.Infof("applyied last attestation slot mutations in %v", time.Since(mutStart)) } } - } - } - } - - for attestedSlot, inclusions := range attestationsBySlot { - mutInclusionSlot := gcp_bigtable.NewMutation() - mutLastAttestationSlot := gcp_bigtable.NewMutation() - mutLastAttestationSlotSet := false - bigtable.lastAttestationCacheMux.Lock() - for validator, inclusionSlot := range inclusions { - mutInclusionSlot.Set(ATTESTATIONS_FAMILY, fmt.Sprintf("%d", validator), gcp_bigtable.Timestamp((max_block_number-inclusionSlot)*1000), []byte{}) + if len(mutsInclusionSlot) == MAX_BATCH_MUTATIONS { + attstart := time.Now() + errs, err := bigtable.tableValidatorsHistory.ApplyBulk(ctx, keysInclusionSlot, mutsInclusionSlot) + if err != nil { + return err + } + for _, err := range errs { + return err + } + logger.Infof("applied attestation mutations in %v", time.Since(attstart)) + mutsInclusionSlot = make([]*gcp_bigtable.Mutation, 0, MAX_BATCH_MUTATIONS) + keysInclusionSlot = make([]string, 0, MAX_BATCH_MUTATIONS) + } - if attestedSlot > bigtable.lastAttestationCache[validator] { - mutLastAttestationSlot.Set(ATTESTATIONS_FAMILY, fmt.Sprintf("%d", validator), gcp_bigtable.Timestamp((attestedSlot)*1000), []byte{}) - bigtable.lastAttestationCache[validator] = attestedSlot - mutLastAttestationSlotSet = true } - + bigtable.lastAttestationCacheMux.Unlock() } - bigtable.lastAttestationCacheMux.Unlock() + } - err := bigtable.tableBeaconchain.Apply(ctx, fmt.Sprintf("%s:e:%s:s:%s", bigtable.chainId, reversedPaddedEpoch(attestedSlot/utils.Config.Chain.ClConfig.SlotsPerEpoch), reversedPaddedSlot(attestedSlot)), mutInclusionSlot) + if len(mutsInclusionSlot) > 0 { + logger.Infof("exporting remaining %v attestation mutations", len(mutsInclusionSlot)) + attstart := time.Now() + errs, err := bigtable.tableValidatorsHistory.ApplyBulk(ctx, keysInclusionSlot, mutsInclusionSlot) if err != nil { return err } + for _, err := range errs { + return err + } + logger.Infof("applied attestation mutations in %v", time.Since(attstart)) + } - if mutLastAttestationSlotSet { - err = bigtable.tableValidators.Apply(ctx, fmt.Sprintf("%s:lastAttestationSlot", bigtable.chainId), mutLastAttestationSlot) - if err != nil { - return err - } + if mutLastAttestationSlotCount > 0 { + err := bigtable.tableValidators.Apply(ctx, fmt.Sprintf("%s:lastAttestationSlot", bigtable.chainId), mutLastAttestationSlot) + if err != nil { + return fmt.Errorf("error applying last attestation slot mutations: %v", err) } } - logger.Infof("exported attestations (new) to bigtable in %v", time.Since(start)) + logger.Infof("exported %v attestations to bigtable in %v", writes, time.Since(start)) return nil } @@ -642,6 +736,9 @@ func (bigtable *Bigtable) SaveProposals(blocks map[uint64]map[string]*types.Bloc return slots[i] < slots[j] }) + muts := make([]*gcp_bigtable.Mutation, 0, MAX_BATCH_MUTATIONS) + keys := make([]string, 0, MAX_BATCH_MUTATIONS) + for _, slot := range slots { for _, b := range blocks[slot] { @@ -649,162 +746,222 @@ func (bigtable *Bigtable) SaveProposals(blocks map[uint64]map[string]*types.Bloc continue } mut := gcp_bigtable.NewMutation() - mut.Set(PROPOSALS_FAMILY, fmt.Sprintf("%d", b.Proposer), gcp_bigtable.Timestamp((max_block_number-b.Slot)*1000), []byte{}) - err := bigtable.tableBeaconchain.Apply(ctx, fmt.Sprintf("%s:e:%s:s:%s", bigtable.chainId, reversedPaddedEpoch(b.Slot/utils.Config.Chain.ClConfig.SlotsPerEpoch), reversedPaddedSlot(b.Slot)), mut) - if err != nil { - return err + mut.Set(PROPOSALS_FAMILY, "b", gcp_bigtable.Timestamp((MAX_CL_BLOCK_NUMBER-b.Slot)*1000), []byte{}) + key := fmt.Sprintf("%s:%s:%s:%s:%s", bigtable.chainId, bigtable.validatorIndexToKey(b.Proposer), PROPOSALS_FAMILY, bigtable.reversedPaddedEpoch(utils.EpochOfSlot(b.Slot)), bigtable.reversedPaddedSlot(b.Slot)) + + muts = append(muts, mut) + keys = append(keys, key) + + if len(muts) == MAX_BATCH_MUTATIONS { + errs, err := bigtable.tableValidatorsHistory.ApplyBulk(ctx, keys, muts) + + if err != nil { + return err + } + + for _, err := range errs { + return err + } + muts = make([]*gcp_bigtable.Mutation, 0, MAX_BATCH_MUTATIONS) + keys = make([]string, 0, MAX_BATCH_MUTATIONS) } } } + + if len(muts) > 0 { + errs, err := bigtable.tableValidatorsHistory.ApplyBulk(ctx, keys, muts) + + if err != nil { + return err + } + + for _, err := range errs { + return err + } + } logger.Infof("exported proposals to bigtable in %v", time.Since(start)) return nil } -func (bigtable *Bigtable) SaveSyncComitteeDuties(blocks map[uint64]map[string]*types.Block) error { +func (bigtable *Bigtable) SaveSyncComitteeDuties(duties map[types.Slot]map[types.ValidatorIndex]bool) error { - ctx, cancel := context.WithTimeout(context.Background(), time.Second*30) + ctx, cancel := context.WithTimeout(context.Background(), time.Minute*2) defer cancel() start := time.Now() - dutiesBySlot := make(map[uint64]map[uint64]bool) //map[dutiesSlot]map[validator]bool - - slots := make([]uint64, 0, len(blocks)) - for slot := range blocks { - slots = append(slots, slot) - } - sort.Slice(slots, func(i, j int) bool { - return slots[i] < slots[j] - }) - - for _, slot := range slots { - for _, b := range blocks[slot] { - if b.Status == 2 { - continue - } else if b.SyncAggregate != nil && len(b.SyncAggregate.SyncCommitteeValidators) > 0 { - bitLen := len(b.SyncAggregate.SyncCommitteeBits) * 8 - valLen := len(b.SyncAggregate.SyncCommitteeValidators) - if bitLen < valLen { - return fmt.Errorf("error getting sync_committee participants: bitLen != valLen: %v != %v", bitLen, valLen) - } - for i, valIndex := range b.SyncAggregate.SyncCommitteeValidators { - if dutiesBySlot[b.Slot] == nil { - dutiesBySlot[b.Slot] = make(map[uint64]bool) - } - dutiesBySlot[b.Slot][valIndex] = utils.BitAtVector(b.SyncAggregate.SyncCommitteeBits, i) - } - } - } - } - - if len(dutiesBySlot) == 0 { + if len(duties) == 0 { logger.Infof("no sync duties to export") return nil } - for slot, validators := range dutiesBySlot { - mut := gcp_bigtable.NewMutation() + + muts := make([]*gcp_bigtable.Mutation, 0, utils.Config.Chain.ClConfig.SlotsPerEpoch*utils.Config.Chain.ClConfig.SyncCommitteeSize+1) + keys := make([]string, 0, utils.Config.Chain.ClConfig.SlotsPerEpoch*utils.Config.Chain.ClConfig.SyncCommitteeSize+1) + + for slot, validators := range duties { + participation := uint64(0) for validator, participated := range validators { + mut := gcp_bigtable.NewMutation() if participated { - mut.Set(SYNC_COMMITTEES_FAMILY, fmt.Sprintf("%d", validator), gcp_bigtable.Timestamp((max_block_number-slot)*1000), []byte{}) + mut.Set(SYNC_COMMITTEES_FAMILY, "s", gcp_bigtable.Timestamp((MAX_CL_BLOCK_NUMBER-slot)*1000), []byte{}) + participation++ } else { - mut.Set(SYNC_COMMITTEES_FAMILY, fmt.Sprintf("%d", validator), gcp_bigtable.Timestamp(0), []byte{}) + mut.Set(SYNC_COMMITTEES_FAMILY, "s", gcp_bigtable.Timestamp(0), []byte{}) } - } - err := bigtable.tableBeaconchain.Apply(ctx, fmt.Sprintf("%s:e:%s:s:%s", bigtable.chainId, reversedPaddedEpoch(slot/utils.Config.Chain.ClConfig.SlotsPerEpoch), reversedPaddedSlot(slot)), mut) + key := fmt.Sprintf("%s:%s:%s:%s:%s", bigtable.chainId, bigtable.validatorIndexToKey(uint64(validator)), SYNC_COMMITTEES_FAMILY, bigtable.reversedPaddedEpoch(utils.EpochOfSlot(uint64(slot))), bigtable.reversedPaddedSlot(uint64(slot))) - if err != nil { - return err + muts = append(muts, mut) + keys = append(keys, key) } + mut := gcp_bigtable.NewMutation() + key := fmt.Sprintf("%s:%s:%s", bigtable.chainId, SYNC_COMMITTEES_PARTICIPATION_FAMILY, bigtable.reversedPaddedSlot(uint64(slot))) + participationEncoded := make([]byte, 8) + binary.LittleEndian.PutUint64(participationEncoded, uint64(participation)) + mut.Set(SYNC_COMMITTEES_PARTICIPATION_FAMILY, "s", gcp_bigtable.Timestamp(0), participationEncoded) + muts = append(muts, mut) + keys = append(keys, key) } - logger.Infof("exported sync committee duties to bigtable in %v", time.Since(start)) - return nil -} -func (bigtable *Bigtable) GetValidatorBalanceHistory(validators []uint64, startEpoch uint64, endEpoch uint64) (map[uint64][]*types.ValidatorBalance, error) { + errs, err := bigtable.tableValidatorsHistory.ApplyBulk(ctx, keys, muts) - valLen := len(validators) - getAllThreshold := 1000 - validatorMap := make(map[uint64]bool, valLen) - for _, validatorIndex := range validators { - validatorMap[validatorIndex] = true + if err != nil { + return err } - ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(time.Second*30)) + for _, err := range errs { + return err + } + + logger.Infof("exported %v sync committee duties to bigtable in %v", len(muts), time.Since(start)) + return nil +} + +// GetMaxValidatorindexForEpoch returns the higest validatorindex with a balance at that epoch +func (bigtable *Bigtable) GetMaxValidatorindexForEpoch(epoch uint64) (uint64, error) { + tmr := time.NewTimer(REPORT_TIMEOUT) + defer tmr.Stop() + go func() { + <-tmr.C + logger.WithFields(logrus.Fields{ + "epoch": epoch, + }).Warnf("%s call took longer than %v", utils.GetCurrentFuncName(), REPORT_TIMEOUT) + }() + + ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(time.Minute*5)) defer cancel() - ranges := bigtable.getEpochRanges(startEpoch, endEpoch) - res := make(map[uint64][]*types.ValidatorBalance, valLen) + key := fmt.Sprintf("%s:%s:%s", bigtable.chainId, VALIDATOR_HIGHEST_ACTIVE_INDEX_FAMILY, bigtable.reversedPaddedEpoch(epoch)) - columnFilters := []gcp_bigtable.Filter{} - if valLen < getAllThreshold { - columnFilters = make([]gcp_bigtable.Filter, 0, valLen) - for _, validator := range validators { - columnFilters = append(columnFilters, gcp_bigtable.ColumnFilter(fmt.Sprintf("%d", validator))) - } + row, err := bigtable.tableValidatorsHistory.ReadRow(ctx, key) + if err != nil { + return 0, err } - filter := gcp_bigtable.ChainFilters( - gcp_bigtable.FamilyFilter(VALIDATOR_BALANCES_FAMILY), - gcp_bigtable.InterleaveFilters(columnFilters...), - ) - - if len(columnFilters) == 1 { // special case to retrieve data for one validators - filter = gcp_bigtable.ChainFilters( - gcp_bigtable.FamilyFilter(VALIDATOR_BALANCES_FAMILY), - columnFilters[0], - ) + for _, ri := range row[VALIDATOR_HIGHEST_ACTIVE_INDEX_FAMILY] { + return binary.LittleEndian.Uint64(ri.Value), nil } - if len(columnFilters) == 0 { // special case to retrieve data for all validators - filter = gcp_bigtable.FamilyFilter(VALIDATOR_BALANCES_FAMILY) + + return 0, nil +} + +func (bigtable *Bigtable) GetValidatorBalanceHistory(validators []uint64, startEpoch uint64, endEpoch uint64) (map[uint64][]*types.ValidatorBalance, error) { + tmr := time.NewTimer(REPORT_TIMEOUT) + defer tmr.Stop() + go func() { + <-tmr.C + logger.WithFields(logrus.Fields{ + "validators_count": len(validators), + "startEpoch": startEpoch, + "endEpoch": endEpoch, + }).Warnf("%s call took longer than %v", utils.GetCurrentFuncName(), REPORT_TIMEOUT) + }() + + if len(validators) == 0 { + return nil, fmt.Errorf("passing empty validator array is unsupported") } - handleRow := func(r gcp_bigtable.Row) bool { - keySplit := strings.Split(r.Key(), ":") + batchSize := 1000 + concurrency := 10 - epoch, err := strconv.ParseUint(keySplit[3], 10, 64) - if err != nil { - logger.Errorf("error parsing epoch from row key %v: %v", r.Key(), err) - return false + ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(time.Minute)) + defer cancel() + + res := make(map[uint64][]*types.ValidatorBalance, len(validators)) + resMux := &sync.Mutex{} + + g, gCtx := errgroup.WithContext(ctx) + g.SetLimit(concurrency) + + for i := 0; i < len(validators); i += batchSize { + + upperBound := i + batchSize + if len(validators) < upperBound { + upperBound = len(validators) } + vals := validators[i:upperBound] - for _, ri := range r[VALIDATOR_BALANCES_FAMILY] { - validator, err := strconv.ParseUint(strings.TrimPrefix(ri.Column, VALIDATOR_BALANCES_FAMILY+":"), 10, 64) - if err != nil { - logger.Errorf("error parsing validator from column key %v: %v", ri.Column, err) - return false + g.Go(func() error { + select { + case <-gCtx.Done(): + return gCtx.Err() + default: } + ranges := bigtable.getValidatorsEpochRanges(vals, VALIDATOR_BALANCES_FAMILY, startEpoch, endEpoch) + ro := gcp_bigtable.LimitRows(int64(endEpoch-startEpoch+1) * int64(len(vals))) - // If we requested more than getAllThreshold validators we will - // get data for all validators and need to filter out all - // unwanted ones - if valLen >= getAllThreshold && !validatorMap[validator] { - continue - } + handleRow := func(r gcp_bigtable.Row) bool { + keySplit := strings.Split(r.Key(), ":") + + epoch, err := strconv.ParseUint(keySplit[3], 10, 64) + if err != nil { + logger.Errorf("error parsing epoch from row key %v: %v", r.Key(), err) + return false + } + + validator, err := bigtable.validatorKeyToIndex(keySplit[1]) + if err != nil { + logger.Errorf("error parsing validator index from row key %v: %v", r.Key(), err) + return false + } + resMux.Lock() + if res[validator] == nil { + res[validator] = make([]*types.ValidatorBalance, 0) + } + resMux.Unlock() + + for _, ri := range r[VALIDATOR_BALANCES_FAMILY] { + + balances := ri.Value - balances := ri.Value + balanceBytes := balances[0:8] + effectiveBalanceBytes := balances[8:16] + balance := binary.LittleEndian.Uint64(balanceBytes) + effectiveBalance := binary.LittleEndian.Uint64(effectiveBalanceBytes) - balanceBytes := balances[0:8] - effectiveBalanceBytes := balances[8:16] - balance := binary.LittleEndian.Uint64(balanceBytes) - effectiveBalance := binary.LittleEndian.Uint64(effectiveBalanceBytes) + resMux.Lock() + res[validator] = append(res[validator], &types.ValidatorBalance{ + Epoch: MAX_EPOCH - epoch, + Balance: balance, + EffectiveBalance: effectiveBalance, + Index: validator, + PublicKey: []byte{}, + }) + resMux.Unlock() + } + return true + } - if res[validator] == nil { - res[validator] = make([]*types.ValidatorBalance, 0) + err := bigtable.tableValidatorsHistory.ReadRows(gCtx, ranges, handleRow, ro) + if err != nil { + return err } - res[validator] = append(res[validator], &types.ValidatorBalance{ - Epoch: max_epoch - epoch, - Balance: balance, - EffectiveBalance: effectiveBalance, - Index: validator, - PublicKey: []byte{}, - }) - } - return true + // logrus.Infof("retrieved data for validators %v - %v", vals[0], vals[len(vals)-1]) + return nil + }) } - err := bigtable.tableBeaconchain.ReadRows(ctx, ranges, handleRow, gcp_bigtable.RowFilter(filter)) - if err != nil { + if err := g.Wait(); err != nil { return nil, err } @@ -812,111 +969,187 @@ func (bigtable *Bigtable) GetValidatorBalanceHistory(validators []uint64, startE } func (bigtable *Bigtable) GetValidatorAttestationHistory(validators []uint64, startEpoch uint64, endEpoch uint64) (map[uint64][]*types.ValidatorAttestation, error) { - valLen := len(validators) + tmr := time.NewTimer(REPORT_TIMEOUT) + defer tmr.Stop() + go func() { + <-tmr.C + logger.WithFields(logrus.Fields{ + "validatorsCount": len(validators), + "startEpoch": startEpoch, + "endEpoch": endEpoch, + }).Warnf("%s call took longer than %v", utils.GetCurrentFuncName(), REPORT_TIMEOUT) + }() + + if len(validators) == 0 { + return nil, fmt.Errorf("passing empty validator array is unsupported") + } + + batchSize := 1000 + concurrency := 10 ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(time.Minute*5)) defer cancel() - slots := []uint64{} + res := make(map[uint64][]*types.ValidatorAttestation, len(validators)) + resMux := &sync.Mutex{} - for slot := startEpoch * utils.Config.Chain.ClConfig.SlotsPerEpoch; slot < (endEpoch+1)*utils.Config.Chain.ClConfig.SlotsPerEpoch; slot++ { - slots = append(slots, slot) - } - orphanedSlotsMap, err := GetOrphanedSlotsMap(slots) - if err != nil { - return nil, err - } + filter := gcp_bigtable.LatestNFilter(1) - ranges := bigtable.getSlotRanges(startEpoch, endEpoch) - res := make(map[uint64][]*types.ValidatorAttestation, len(validators)) + g, gCtx := errgroup.WithContext(ctx) + g.SetLimit(concurrency) - columnFilters := []gcp_bigtable.Filter{} - if valLen < 1000 { - columnFilters = make([]gcp_bigtable.Filter, 0, len(validators)) - for _, validator := range validators { - columnFilters = append(columnFilters, gcp_bigtable.ColumnFilter(fmt.Sprintf("%d", validator))) + attestationsMap := make(map[types.ValidatorIndex]map[types.Slot][]*types.ValidatorAttestation) + + for i := 0; i < len(validators); i += batchSize { + + upperBound := i + batchSize + if len(validators) < upperBound { + upperBound = len(validators) } + vals := validators[i:upperBound] + + g.Go(func() error { + select { + case <-gCtx.Done(): + return gCtx.Err() + default: + } + ranges := bigtable.getValidatorsEpochRanges(vals, ATTESTATIONS_FAMILY, startEpoch, endEpoch) + err := bigtable.tableValidatorsHistory.ReadRows(ctx, ranges, func(r gcp_bigtable.Row) bool { + keySplit := strings.Split(r.Key(), ":") + + validator, err := bigtable.validatorKeyToIndex(keySplit[1]) + if err != nil { + logger.Errorf("error parsing validator from row key %v: %v", r.Key(), err) + return false + } + + for _, ri := range r[ATTESTATIONS_FAMILY] { + attesterSlotString := strings.Replace(ri.Column, ATTESTATIONS_FAMILY+":", "", 1) + attesterSlot, err := strconv.ParseUint(attesterSlotString, 10, 64) + if err != nil { + logger.Errorf("error parsing slot from row key %v: %v", r.Key(), err) + return false + } + inclusionSlot := MAX_CL_BLOCK_NUMBER - uint64(ri.Timestamp)/1000 + + status := uint64(1) + if inclusionSlot == MAX_CL_BLOCK_NUMBER { + inclusionSlot = 0 + status = 0 + } + + resMux.Lock() + if attestationsMap[types.ValidatorIndex(validator)] == nil { + attestationsMap[types.ValidatorIndex(validator)] = make(map[types.Slot][]*types.ValidatorAttestation) + } + + if attestationsMap[types.ValidatorIndex(validator)][types.Slot(attesterSlot)] == nil { + attestationsMap[types.ValidatorIndex(validator)][types.Slot(attesterSlot)] = make([]*types.ValidatorAttestation, 0) + } + + attestationsMap[types.ValidatorIndex(validator)][types.Slot(attesterSlot)] = append(attestationsMap[types.ValidatorIndex(validator)][types.Slot(attesterSlot)], &types.ValidatorAttestation{ + InclusionSlot: inclusionSlot, + Status: status, + }) + resMux.Unlock() + + } + return true + }, gcp_bigtable.RowFilter(filter)) + + return err + }) } - filter := gcp_bigtable.ChainFilters( - gcp_bigtable.FamilyFilter(ATTESTATIONS_FAMILY), - gcp_bigtable.InterleaveFilters(columnFilters...), - gcp_bigtable.LatestNFilter(1), - ) + if err := g.Wait(); err != nil { + return nil, err + } - if len(columnFilters) == 1 { // special case to retrieve data for one validators - filter = gcp_bigtable.ChainFilters( - gcp_bigtable.FamilyFilter(ATTESTATIONS_FAMILY), - columnFilters[0], - gcp_bigtable.LatestNFilter(1), - ) + // Find all missed and orphaned slots + slots := []uint64{} + maxSlot := ((endEpoch + 1) * utils.Config.Chain.ClConfig.SlotsPerEpoch) - 1 + for slot := startEpoch * utils.Config.Chain.ClConfig.SlotsPerEpoch; slot <= maxSlot; slot++ { + slots = append(slots, slot) } - if len(columnFilters) == 0 { // special case to retrieve data for all validators - filter = gcp_bigtable.ChainFilters( - gcp_bigtable.FamilyFilter(ATTESTATIONS_FAMILY), - gcp_bigtable.LatestNFilter(1), - ) + + var missedSlotsMap map[uint64]bool + var orphanedSlotsMap map[uint64]bool + + g = new(errgroup.Group) + + g.Go(func() error { + var err error + missedSlotsMap, err = GetMissedSlotsMap(slots) + return err + }) + + g.Go(func() error { + var err error + orphanedSlotsMap, err = GetOrphanedSlotsMap(slots) + return err + }) + err := g.Wait() + if err != nil { + return nil, err } - err = bigtable.tableBeaconchain.ReadRows(ctx, ranges, func(r gcp_bigtable.Row) bool { - keySplit := strings.Split(r.Key(), ":") - attesterSlot, err := strconv.ParseUint(keySplit[4], 10, 64) - if err != nil { - logger.Errorf("error parsing slot from row key %v: %v", r.Key(), err) - return false + // Convert the attestationsMap info to the return format + // Set the delay of the inclusionSlot + for validator, attestations := range attestationsMap { + if res[uint64(validator)] == nil { + res[uint64(validator)] = make([]*types.ValidatorAttestation, 0) } - attesterSlot = max_block_number - attesterSlot - for _, ri := range r[ATTESTATIONS_FAMILY] { - inclusionSlot := max_block_number - uint64(ri.Timestamp)/1000 - - status := uint64(1) - if inclusionSlot == max_block_number { - inclusionSlot = 0 - status = 0 - } else if orphanedSlotsMap[inclusionSlot] { - status = 0 - } - - validator, err := strconv.ParseUint(strings.TrimPrefix(ri.Column, ATTESTATIONS_FAMILY+":"), 10, 64) - if err != nil { - logger.Errorf("error parsing validator from column key %v: %v", ri.Column, err) - return false - } + for attesterSlot, att := range attestations { + currentAttInfo := att[0] + for _, attInfo := range att { + if orphanedSlotsMap[attInfo.InclusionSlot] { + attInfo.Status = 0 + } - if res[validator] == nil { - res[validator] = make([]*types.ValidatorAttestation, 0) + if currentAttInfo.Status != 1 && attInfo.Status == 1 { + currentAttInfo.Status = attInfo.Status + currentAttInfo.InclusionSlot = attInfo.InclusionSlot + } } - if len(res[validator]) > 0 && res[validator][len(res[validator])-1].AttesterSlot == attesterSlot { - // don't override successful attestion, that was included in a different slot - if status == 1 || res[validator][len(res[validator])-1].Status != 1 { - res[validator][len(res[validator])-1].InclusionSlot = inclusionSlot - res[validator][len(res[validator])-1].Status = status - res[validator][len(res[validator])-1].Delay = int64(inclusionSlot - attesterSlot) + missedSlotsCount := uint64(0) + for slot := uint64(attesterSlot) + 1; slot < currentAttInfo.InclusionSlot; slot++ { + if missedSlotsMap[slot] || orphanedSlotsMap[slot] { + missedSlotsCount++ } - } else { - res[validator] = append(res[validator], &types.ValidatorAttestation{ - Index: validator, - Epoch: attesterSlot / utils.Config.Chain.ClConfig.SlotsPerEpoch, - AttesterSlot: attesterSlot, - CommitteeIndex: 0, - Status: status, - InclusionSlot: inclusionSlot, - Delay: int64(inclusionSlot) - int64(attesterSlot) - 1, - }) } + currentAttInfo.Index = uint64(validator) + currentAttInfo.Epoch = uint64(attesterSlot) / utils.Config.Chain.ClConfig.SlotsPerEpoch + currentAttInfo.CommitteeIndex = 0 + currentAttInfo.AttesterSlot = uint64(attesterSlot) + currentAttInfo.Delay = int64(currentAttInfo.InclusionSlot - uint64(attesterSlot) - missedSlotsCount - 1) + res[uint64(validator)] = append(res[uint64(validator)], currentAttInfo) } - return true - }, gcp_bigtable.RowFilter(filter)) - if err != nil { - return nil, err + } + + // Sort the result by attesterSlot desc + for validator, att := range res { + sort.Slice(att, func(i, j int) bool { + return att[i].AttesterSlot > att[j].AttesterSlot + }) + res[validator] = att } return res, nil } func (bigtable *Bigtable) GetLastAttestationSlots(validators []uint64) (map[uint64]uint64, error) { + tmr := time.NewTimer(REPORT_TIMEOUT) + defer tmr.Stop() + go func() { + <-tmr.C + logger.WithFields(logrus.Fields{ + "validatorsCount": len(validators), + }).Warnf("%s call took longer than %v", utils.GetCurrentFuncName(), REPORT_TIMEOUT) + }() + valLen := len(validators) ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(time.Minute*5)) @@ -972,8 +1205,66 @@ func (bigtable *Bigtable) GetLastAttestationSlots(validators []uint64) (map[uint return res, nil } +func (bigtable *Bigtable) GetSyncParticipationBySlotRange(startSlot, endSlot uint64) (map[uint64]uint64, error) { + tmr := time.NewTimer(REPORT_TIMEOUT) + defer tmr.Stop() + go func() { + <-tmr.C + logger.WithFields(logrus.Fields{ + "startSlot": startSlot, + "endSlot": endSlot, + }).Warnf("%s call took longer than %v", utils.GetCurrentFuncName(), REPORT_TIMEOUT) + }() + + ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(time.Minute*5)) + defer cancel() + + startKey := fmt.Sprintf("%s:%s:%s", bigtable.chainId, SYNC_COMMITTEES_PARTICIPATION_FAMILY, bigtable.reversedPaddedSlot(endSlot)) + endKey := fmt.Sprintf("%s:%s:%s\x00", bigtable.chainId, SYNC_COMMITTEES_PARTICIPATION_FAMILY, bigtable.reversedPaddedSlot(startSlot)) + rowRange := gcp_bigtable.NewRange(startKey, endKey) + + res := make(map[uint64]uint64) + err := bigtable.tableValidatorsHistory.ReadRows(ctx, rowRange, func(r gcp_bigtable.Row) bool { + keySplit := strings.Split(r.Key(), ":") + + slot, err := strconv.ParseUint(keySplit[2], 10, 64) + if err != nil { + logger.Errorf("error parsing epoch from row key %v: %v", r.Key(), err) + return false + } + slot = MAX_CL_BLOCK_NUMBER - slot + + for _, ri := range r[SYNC_COMMITTEES_PARTICIPATION_FAMILY] { + res[slot] = binary.LittleEndian.Uint64(ri.Value) + } + + return true + }) + if err != nil { + return nil, err + } + + return res, nil +} + func (bigtable *Bigtable) GetValidatorMissedAttestationHistory(validators []uint64, startEpoch uint64, endEpoch uint64) (map[uint64]map[uint64]bool, error) { - valLen := len(validators) + tmr := time.NewTimer(REPORT_TIMEOUT) + defer tmr.Stop() + go func() { + <-tmr.C + logger.WithFields(logrus.Fields{ + "validatorsCount": len(validators), + "startEpoch": startEpoch, + "endEpoch": endEpoch, + }).Warnf("%s call took longer than %v", utils.GetCurrentFuncName(), REPORT_TIMEOUT) + }() + + if len(validators) == 0 { + return nil, fmt.Errorf("passing empty validator array is unsupported") + } + + batchSize := 1000 + concurrency := 10 ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(time.Minute*20)) defer cancel() @@ -988,175 +1279,182 @@ func (bigtable *Bigtable) GetValidatorMissedAttestationHistory(validators []uint return nil, err } - ranges := bigtable.getSlotRanges(startEpoch, endEpoch) - res := make(map[uint64]map[uint64]bool) foundValid := make(map[uint64]map[uint64]bool) - columnFilters := []gcp_bigtable.Filter{} - if valLen < 1000 { - columnFilters = make([]gcp_bigtable.Filter, 0, len(validators)) - for _, validator := range validators { - columnFilters = append(columnFilters, gcp_bigtable.ColumnFilter(fmt.Sprintf("%d", validator))) - } - } + resMux := &sync.Mutex{} - filter := gcp_bigtable.ChainFilters( - gcp_bigtable.FamilyFilter(ATTESTATIONS_FAMILY), - gcp_bigtable.InterleaveFilters(columnFilters...), - gcp_bigtable.LatestNFilter(1), - ) + filter := gcp_bigtable.LatestNFilter(1) - if len(columnFilters) == 1 { // special case to retrieve data for one validators - filter = gcp_bigtable.ChainFilters( - gcp_bigtable.FamilyFilter(ATTESTATIONS_FAMILY), - columnFilters[0], - gcp_bigtable.LatestNFilter(1), - ) - } - if len(columnFilters) == 0 { // special case to retrieve data for all validators - filter = gcp_bigtable.ChainFilters( - gcp_bigtable.FamilyFilter(ATTESTATIONS_FAMILY), - gcp_bigtable.LatestNFilter(1), - ) - } + g, gCtx := errgroup.WithContext(ctx) + g.SetLimit(concurrency) - err = bigtable.tableBeaconchain.ReadRows(ctx, ranges, func(r gcp_bigtable.Row) bool { - keySplit := strings.Split(r.Key(), ":") + for i := 0; i < len(validators); i += batchSize { - attesterSlot, err := strconv.ParseUint(keySplit[4], 10, 64) - if err != nil { - logger.Errorf("error parsing slot from row key %v: %v", r.Key(), err) - return false + upperBound := i + batchSize + if len(validators) < upperBound { + upperBound = len(validators) } - attesterSlot = max_block_number - attesterSlot - - for _, ri := range r[ATTESTATIONS_FAMILY] { - inclusionSlot := max_block_number - uint64(ri.Timestamp)/1000 - - status := uint64(1) - if inclusionSlot == max_block_number { - status = 0 - } + vals := validators[i:upperBound] - validator, err := strconv.ParseUint(strings.TrimPrefix(ri.Column, ATTESTATIONS_FAMILY+":"), 10, 64) - if err != nil { - logger.Errorf("error parsing validator from column key %v: %v", ri.Column, err) - return false + g.Go(func() error { + select { + case <-gCtx.Done(): + return gCtx.Err() + default: } + ranges := bigtable.getValidatorsEpochRanges(vals, ATTESTATIONS_FAMILY, startEpoch, endEpoch) + err = bigtable.tableValidatorsHistory.ReadRows(ctx, ranges, func(r gcp_bigtable.Row) bool { + keySplit := strings.Split(r.Key(), ":") - // only if the attestation was not included in another slot we count it as missed - if (status == 0 || orphanedSlotsMap[inclusionSlot]) && (foundValid[validator] == nil || !foundValid[validator][attesterSlot]) { - if res[validator] == nil { - res[validator] = make(map[uint64]bool, 0) - } - res[validator][attesterSlot] = true - } else { - if res[validator] != nil && res[validator][attesterSlot] { - delete(res[validator], attesterSlot) + validator, err := bigtable.validatorKeyToIndex(keySplit[1]) + if err != nil { + logger.Errorf("error parsing validator from row key %v: %v", r.Key(), err) + return false } - if foundValid[validator] == nil { - foundValid[validator] = make(map[uint64]bool, 0) + + for _, ri := range r[ATTESTATIONS_FAMILY] { + attesterSlotString := strings.Replace(ri.Column, ATTESTATIONS_FAMILY+":", "", 1) + attesterSlot, err := strconv.ParseUint(attesterSlotString, 10, 64) + if err != nil { + logger.Errorf("error parsing slot from row key %v: %v", r.Key(), err) + return false + } + + inclusionSlot := MAX_CL_BLOCK_NUMBER - uint64(ri.Timestamp)/1000 + + status := uint64(1) + if inclusionSlot == MAX_CL_BLOCK_NUMBER { + status = 0 + } + + resMux.Lock() + // only if the attestation was not included in another slot we count it as missed + if (status == 0 || orphanedSlotsMap[inclusionSlot]) && (foundValid[validator] == nil || !foundValid[validator][attesterSlot]) { + if res[validator] == nil { + res[validator] = make(map[uint64]bool, 0) + } + res[validator][attesterSlot] = true + } else { + if res[validator] != nil && res[validator][attesterSlot] { + delete(res[validator], attesterSlot) + } + if foundValid[validator] == nil { + foundValid[validator] = make(map[uint64]bool, 0) + } + foundValid[validator][attesterSlot] = true + } + resMux.Unlock() } - foundValid[validator][attesterSlot] = true - } - } - return true - }, gcp_bigtable.RowFilter(filter)) - if err != nil { + return true + }, gcp_bigtable.RowFilter(filter)) + + return err + }) + } + + if err := g.Wait(); err != nil { return nil, err } return res, nil } -func (bigtable *Bigtable) GetValidatorSyncDutiesHistoryOrdered(validators []uint64, startEpoch uint64, endEpoch uint64, reverseOrdering bool) (map[uint64][]*types.ValidatorSyncParticipation, error) { - res, err := bigtable.GetValidatorSyncDutiesHistory(validators, startEpoch, endEpoch) - if err != nil { - return nil, err - } - if reverseOrdering { - for _, duties := range res { - utils.ReverseSlice(duties) - } +func (bigtable *Bigtable) GetValidatorSyncDutiesHistory(validators []uint64, startSlot uint64, endSlot uint64) (map[uint64]map[uint64]*types.ValidatorSyncParticipation, error) { + tmr := time.NewTimer(REPORT_TIMEOUT) + defer tmr.Stop() + go func() { + <-tmr.C + logger.WithFields(logrus.Fields{ + "validatorsCount": len(validators), + "startSlot": startSlot, + "endSlot": endSlot, + }).Warnf("%s call took longer than %v", utils.GetCurrentFuncName(), REPORT_TIMEOUT) + }() + + if len(validators) == 0 { + return nil, fmt.Errorf("passing empty validator array is unsupported") } - return res, nil -} -func (bigtable *Bigtable) GetValidatorSyncDutiesHistory(validators []uint64, startEpoch uint64, endEpoch uint64) (map[uint64][]*types.ValidatorSyncParticipation, error) { + batchSize := 1000 + concurrency := 10 + ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(time.Minute*5)) defer cancel() - ranges := bigtable.getSlotRanges(startEpoch, endEpoch) - res := make(map[uint64][]*types.ValidatorSyncParticipation, len(validators)) + res := make(map[uint64]map[uint64]*types.ValidatorSyncParticipation, len(validators)) + resMux := &sync.Mutex{} - columnFilters := make([]gcp_bigtable.Filter, 0, len(validators)) - for _, validator := range validators { - columnFilters = append(columnFilters, gcp_bigtable.ColumnFilter(fmt.Sprintf("%d", validator))) - } + filter := gcp_bigtable.LatestNFilter(1) - filter := gcp_bigtable.ChainFilters( - gcp_bigtable.FamilyFilter(SYNC_COMMITTEES_FAMILY), - gcp_bigtable.InterleaveFilters(columnFilters...), - gcp_bigtable.LatestNFilter(1), - ) - - if len(columnFilters) == 1 { // special case to retrieve data for one validators - filter = gcp_bigtable.ChainFilters( - gcp_bigtable.FamilyFilter(SYNC_COMMITTEES_FAMILY), - columnFilters[0], - gcp_bigtable.LatestNFilter(1), - ) - } - if len(columnFilters) == 0 { // special case to retrieve data for all validators - filter = gcp_bigtable.ChainFilters( - gcp_bigtable.FamilyFilter(SYNC_COMMITTEES_FAMILY), - gcp_bigtable.LatestNFilter(1), - ) - } + g, gCtx := errgroup.WithContext(ctx) + g.SetLimit(concurrency) - err := bigtable.tableBeaconchain.ReadRows(ctx, ranges, func(r gcp_bigtable.Row) bool { + for i := 0; i < len(validators); i += batchSize { - for _, ri := range r[SYNC_COMMITTEES_FAMILY] { - keySplit := strings.Split(r.Key(), ":") + upperBound := i + batchSize + if len(validators) < upperBound { + upperBound = len(validators) + } + vals := validators[i:upperBound] - slot, err := strconv.ParseUint(keySplit[4], 10, 64) - if err != nil { - logger.Errorf("error parsing slot from row key %v: %v", r.Key(), err) - return false + g.Go(func() error { + select { + case <-gCtx.Done(): + return gCtx.Err() + default: } - slot = max_block_number - slot - inclusionSlot := max_block_number - uint64(ri.Timestamp)/1000 + ranges := bigtable.getValidatorSlotRanges(vals, SYNC_COMMITTEES_FAMILY, startSlot, endSlot) - status := uint64(1) // 1: participated - if inclusionSlot == max_block_number { - inclusionSlot = 0 - status = 0 // 0: missed - } + err := bigtable.tableValidatorsHistory.ReadRows(ctx, ranges, func(r gcp_bigtable.Row) bool { + keySplit := strings.Split(r.Key(), ":") - validator, err := strconv.ParseUint(strings.TrimPrefix(ri.Column, SYNC_COMMITTEES_FAMILY+":"), 10, 64) - if err != nil { - logger.Errorf("error parsing validator from column key %v: %v", ri.Column, err) - return false - } + validator, err := bigtable.validatorKeyToIndex(keySplit[1]) + if err != nil { + logger.Errorf("error parsing validator from row key %v: %v", r.Key(), err) + return false + } + slot, err := strconv.ParseUint(keySplit[4], 10, 64) + if err != nil { + logger.Errorf("error parsing slot from row key %v: %v", r.Key(), err) + return false + } + slot = MAX_CL_BLOCK_NUMBER - slot - if res[validator] == nil { - res[validator] = make([]*types.ValidatorSyncParticipation, 0) - } + for _, ri := range r[SYNC_COMMITTEES_FAMILY] { - if len(res[validator]) > 0 && res[validator][len(res[validator])-1].Slot == slot { - res[validator][len(res[validator])-1].Status = status - } else { - res[validator] = append(res[validator], &types.ValidatorSyncParticipation{ - Slot: slot, - Status: status, - }) - } + inclusionSlot := MAX_CL_BLOCK_NUMBER - uint64(ri.Timestamp)/1000 - } - return true - }, gcp_bigtable.RowFilter(filter)) - if err != nil { + status := uint64(1) // 1: participated + if inclusionSlot == MAX_CL_BLOCK_NUMBER { + inclusionSlot = 0 + status = 0 // 0: missed + } + + resMux.Lock() + if res[validator] == nil { + res[validator] = make(map[uint64]*types.ValidatorSyncParticipation, 0) + } + + if len(res[validator]) > 0 && res[validator][slot] != nil { + res[validator][slot].Status = status + } else { + res[validator][slot] = &types.ValidatorSyncParticipation{ + Slot: slot, + Status: status, + } + } + resMux.Unlock() + + } + return true + }, gcp_bigtable.RowFilter(filter)) + + return err + }) + } + + if err := g.Wait(); err != nil { return nil, err } @@ -1164,6 +1462,17 @@ func (bigtable *Bigtable) GetValidatorSyncDutiesHistory(validators []uint64, sta } func (bigtable *Bigtable) GetValidatorMissedAttestationsCount(validators []uint64, firstEpoch uint64, lastEpoch uint64) (map[uint64]*types.ValidatorMissedAttestationsStatistic, error) { + tmr := time.NewTimer(REPORT_TIMEOUT) + defer tmr.Stop() + go func() { + <-tmr.C + logger.WithFields(logrus.Fields{ + "validatorsCount": len(validators), + "startEpoch": firstEpoch, + "endEpoch": lastEpoch, + }).Warnf("%s call took longer than %v", utils.GetCurrentFuncName(), REPORT_TIMEOUT) + }() + if firstEpoch > lastEpoch { return nil, fmt.Errorf("GetValidatorMissedAttestationsCount received an invalid firstEpoch (%d) and lastEpoch (%d) combination", firstEpoch, lastEpoch) } @@ -1192,7 +1501,8 @@ func (bigtable *Bigtable) GetValidatorMissedAttestationsCount(validators []uint6 } func (bigtable *Bigtable) GetValidatorSyncDutiesStatistics(validators []uint64, startEpoch uint64, endEpoch uint64) (map[uint64]*types.ValidatorSyncDutiesStatistic, error) { - data, err := bigtable.GetValidatorSyncDutiesHistory(validators, startEpoch, endEpoch) + + data, err := bigtable.GetValidatorSyncDutiesHistory(validators, startEpoch*utils.Config.Chain.ClConfig.SlotsPerEpoch, ((endEpoch+1)*utils.Config.Chain.ClConfig.SlotsPerEpoch)-1) if err != nil { return nil, err @@ -1244,7 +1554,13 @@ func (bigtable *Bigtable) GetValidatorSyncDutiesStatistics(validators []uint64, // returns the validator attestation effectiveness in % func (bigtable *Bigtable) GetValidatorEffectiveness(validators []uint64, epoch uint64) ([]*types.ValidatorEffectiveness, error) { - data, err := bigtable.GetValidatorAttestationHistory(validators, epoch-99, epoch) + end := epoch + start := uint64(0) + lookback := uint64(99) + if end > lookback { + start = end - lookback + } + data, err := bigtable.GetValidatorAttestationHistory(validators, start, end) if err != nil { return nil, err @@ -1283,9 +1599,17 @@ func (bigtable *Bigtable) GetValidatorEffectiveness(validators []uint64, epoch u return res, nil } -func (bigtable *Bigtable) GetValidatorBalanceStatistics(startEpoch, endEpoch uint64) (map[uint64]*types.ValidatorBalanceStatistic, error) { - ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(time.Minute*10)) - defer cancel() +func (bigtable *Bigtable) GetValidatorBalanceStatistics(validators []uint64, startEpoch, endEpoch uint64) (map[uint64]*types.ValidatorBalanceStatistic, error) { + tmr := time.NewTimer(REPORT_TIMEOUT) + defer tmr.Stop() + go func() { + <-tmr.C + logger.WithFields(logrus.Fields{ + "validatorsCount": len(validators), + "startEpoch": startEpoch, + "endEpoch": endEpoch, + }).Warnf("%s call took longer than %v", utils.GetCurrentFuncName(), REPORT_TIMEOUT) + }() type ResultContainer struct { mu sync.Mutex @@ -1293,175 +1617,169 @@ func (bigtable *Bigtable) GetValidatorBalanceStatistics(startEpoch, endEpoch uin } resultContainer := ResultContainer{} resultContainer.res = make(map[uint64]*types.ValidatorBalanceStatistic) - g, gCtx := errgroup.WithContext(ctx) - batchSize := utilMath.MaxU64(5, (endEpoch-startEpoch)/5) // we can speed up the loading by splitting it up. Making the batchSize even smaller has no effect but increases the memory consumption. - for e := startEpoch; e < endEpoch; e += batchSize { - fromEpoch := e - toEpoch := fromEpoch + batchSize - 1 - if toEpoch > endEpoch { - toEpoch = endEpoch - } - g.Go(func() error { - select { - case <-gCtx.Done(): - return gCtx.Err() - default: - } - ranges := bigtable.getEpochRanges(fromEpoch, toEpoch) - err := bigtable.tableBeaconchain.ReadRows(ctx, ranges, func(r gcp_bigtable.Row) bool { - keySplit := strings.Split(r.Key(), ":") - - epoch, err := strconv.ParseUint(keySplit[3], 10, 64) - if err != nil { - logger.Errorf("error parsing epoch from row key %v: %v", r.Key(), err) - return false - } - epoch = max_epoch - epoch - logger.Infof("retrieved %v balances entries for epoch %v", len(r[VALIDATOR_BALANCES_FAMILY]), epoch) - resultContainer.mu.Lock() - for _, ri := range r[VALIDATOR_BALANCES_FAMILY] { - validator, err := strconv.ParseUint(strings.TrimPrefix(ri.Column, VALIDATOR_BALANCES_FAMILY+":"), 10, 64) - if err != nil { - logger.Errorf("error parsing validator from column key %v: %v", ri.Column, err) - return false - } - balances := ri.Value + // g, gCtx := errgroup.WithContext(ctx) + batchSize := 10000 + // g.SetLimit(1) + for i := 0; i < len(validators); i += batchSize { - balanceBytes := balances[0:8] - effectiveBalanceBytes := balances[8:16] - balance := binary.LittleEndian.Uint64(balanceBytes) - effectiveBalance := binary.LittleEndian.Uint64(effectiveBalanceBytes) - if resultContainer.res[validator] == nil { - resultContainer.res[validator] = &types.ValidatorBalanceStatistic{ - Index: validator, - MinEffectiveBalance: effectiveBalance, - MaxEffectiveBalance: 0, - MinBalance: balance, - MaxBalance: 0, - StartEffectiveBalance: 0, - EndEffectiveBalance: 0, - StartBalance: 0, - EndBalance: 0, - } - } + upperBound := i + batchSize + if len(validators) < upperBound { + upperBound = len(validators) + } + vals := validators[i:upperBound] - if epoch == startEpoch { - resultContainer.res[validator].StartBalance = balance - resultContainer.res[validator].StartEffectiveBalance = effectiveBalance - } + logrus.Infof("retrieving validator balance stats for validators %v - %v", vals[0], vals[len(vals)-1]) - if epoch == endEpoch { - resultContainer.res[validator].EndBalance = balance - resultContainer.res[validator].EndEffectiveBalance = effectiveBalance + res, err := bigtable.GetValidatorBalanceHistory(vals, startEpoch, endEpoch) + if err != nil { + return nil, err + } + resultContainer.mu.Lock() + for validator, balances := range res { + for _, balance := range balances { + if resultContainer.res[validator] == nil { + resultContainer.res[validator] = &types.ValidatorBalanceStatistic{ + Index: validator, + MinEffectiveBalance: balance.EffectiveBalance, + MaxEffectiveBalance: 0, + MinBalance: balance.Balance, + MaxBalance: 0, + StartEffectiveBalance: 0, + EndEffectiveBalance: 0, + StartBalance: 0, + EndBalance: 0, } + } - if balance > resultContainer.res[validator].MaxBalance { - resultContainer.res[validator].MaxBalance = balance - } - if balance < resultContainer.res[validator].MinBalance { - resultContainer.res[validator].MinBalance = balance - } + if balance.Epoch == startEpoch { + resultContainer.res[validator].StartBalance = balance.Balance + resultContainer.res[validator].StartEffectiveBalance = balance.EffectiveBalance + } - if balance > resultContainer.res[validator].MaxEffectiveBalance { - resultContainer.res[validator].MaxEffectiveBalance = balance - } - if balance < resultContainer.res[validator].MinEffectiveBalance { - resultContainer.res[validator].MinEffectiveBalance = balance - } + if balance.Epoch == endEpoch { + resultContainer.res[validator].EndBalance = balance.Balance + resultContainer.res[validator].EndEffectiveBalance = balance.EffectiveBalance } - resultContainer.mu.Unlock() - return true - }, gcp_bigtable.RowFilter(gcp_bigtable.FamilyFilter(VALIDATOR_BALANCES_FAMILY))) + if balance.Balance > resultContainer.res[validator].MaxBalance { + resultContainer.res[validator].MaxBalance = balance.Balance + } + if balance.Balance < resultContainer.res[validator].MinBalance { + resultContainer.res[validator].MinBalance = balance.Balance + } - return err + if balance.EffectiveBalance > resultContainer.res[validator].MaxEffectiveBalance { + resultContainer.res[validator].MaxEffectiveBalance = balance.EffectiveBalance + } + if balance.EffectiveBalance < resultContainer.res[validator].MinEffectiveBalance { + resultContainer.res[validator].MinEffectiveBalance = balance.EffectiveBalance + } + } + } - }) - } + resultContainer.mu.Unlock() - if err := g.Wait(); err != nil { - return nil, err } return resultContainer.res, nil } func (bigtable *Bigtable) GetValidatorProposalHistory(validators []uint64, startEpoch uint64, endEpoch uint64) (map[uint64][]*types.ValidatorProposal, error) { + tmr := time.NewTimer(REPORT_TIMEOUT) + defer tmr.Stop() + go func() { + <-tmr.C + logger.WithFields(logrus.Fields{ + "validatorsCount": len(validators), + "startEpoch": startEpoch, + "endEpoch": endEpoch, + }).Warnf("%s call took longer than %v", utils.GetCurrentFuncName(), REPORT_TIMEOUT) + }() + + if len(validators) == 0 { + return nil, fmt.Errorf("passing empty validator array is unsupported") + } + + batchSize := 1000 + concurrency := 10 + ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(time.Second*30)) defer cancel() - ranges := bigtable.getSlotRanges(startEpoch, endEpoch) res := make(map[uint64][]*types.ValidatorProposal, len(validators)) + resMux := &sync.Mutex{} - columnFilters := make([]gcp_bigtable.Filter, 0, len(validators)) - for _, validator := range validators { - columnFilters = append(columnFilters, gcp_bigtable.ColumnFilter(fmt.Sprintf("%d", validator))) - } + filter := gcp_bigtable.LatestNFilter(1) - filter := gcp_bigtable.ChainFilters( - gcp_bigtable.FamilyFilter(PROPOSALS_FAMILY), - gcp_bigtable.InterleaveFilters(columnFilters...), - gcp_bigtable.LatestNFilter(1), - ) + g, gCtx := errgroup.WithContext(ctx) + g.SetLimit(concurrency) - if len(columnFilters) == 1 { // special case to retrieve data for one validators - filter = gcp_bigtable.ChainFilters( - gcp_bigtable.FamilyFilter(PROPOSALS_FAMILY), - columnFilters[0], - gcp_bigtable.LatestNFilter(1), - ) - } - if len(columnFilters) == 0 { // special case to retrieve data for all validators - filter = gcp_bigtable.ChainFilters( - gcp_bigtable.FamilyFilter(PROPOSALS_FAMILY), - gcp_bigtable.LatestNFilter(1), - ) - } + for i := 0; i < len(validators); i += batchSize { - err := bigtable.tableBeaconchain.ReadRows(ctx, ranges, func(r gcp_bigtable.Row) bool { - for _, ri := range r[PROPOSALS_FAMILY] { - keySplit := strings.Split(r.Key(), ":") + upperBound := i + batchSize + if len(validators) < upperBound { + upperBound = len(validators) + } + vals := validators[i:upperBound] - proposalSlot, err := strconv.ParseUint(keySplit[4], 10, 64) - if err != nil { - logger.Errorf("error parsing slot from row key %v: %v", r.Key(), err) - return false + g.Go(func() error { + select { + case <-gCtx.Done(): + return gCtx.Err() + default: } - proposalSlot = max_block_number - proposalSlot - inclusionSlot := max_block_number - uint64(r[PROPOSALS_FAMILY][0].Timestamp)/1000 + ranges := bigtable.getValidatorsEpochSlotRanges(vals, PROPOSALS_FAMILY, startEpoch, endEpoch) + err := bigtable.tableValidatorsHistory.ReadRows(ctx, ranges, func(r gcp_bigtable.Row) bool { + for _, ri := range r[PROPOSALS_FAMILY] { + keySplit := strings.Split(r.Key(), ":") - status := uint64(1) - if inclusionSlot == max_block_number { - inclusionSlot = 0 - status = 2 - } + proposalSlot, err := strconv.ParseUint(keySplit[4], 10, 64) + if err != nil { + logger.Errorf("error parsing slot from row key %v: %v", r.Key(), err) + return false + } + proposalSlot = MAX_CL_BLOCK_NUMBER - proposalSlot + inclusionSlot := MAX_CL_BLOCK_NUMBER - uint64(r[PROPOSALS_FAMILY][0].Timestamp)/1000 - validator, err := strconv.ParseUint(strings.TrimPrefix(ri.Column, PROPOSALS_FAMILY+":"), 10, 64) - if err != nil { - logger.Errorf("error parsing validator from column key %v: %v", ri.Column, err) - return false - } + status := uint64(1) + if inclusionSlot == MAX_CL_BLOCK_NUMBER { + inclusionSlot = 0 + status = 2 + } - if res[validator] == nil { - res[validator] = make([]*types.ValidatorProposal, 0) - } + validator, err := bigtable.validatorKeyToIndex(keySplit[1]) + if err != nil { + logger.Errorf("error parsing validator from column key %v: %v", ri.Column, err) + return false + } - if len(res[validator]) > 0 && res[validator][len(res[validator])-1].Slot == proposalSlot { - res[validator][len(res[validator])-1].Slot = proposalSlot - res[validator][len(res[validator])-1].Status = status - } else { - res[validator] = append(res[validator], &types.ValidatorProposal{ - Index: validator, - Status: status, - Slot: proposalSlot, - }) - } + resMux.Lock() + if res[validator] == nil { + res[validator] = make([]*types.ValidatorProposal, 0) + } - } - return true - }, gcp_bigtable.RowFilter(filter)) - if err != nil { + if len(res[validator]) > 0 && res[validator][len(res[validator])-1].Slot == proposalSlot { + res[validator][len(res[validator])-1].Slot = proposalSlot + res[validator][len(res[validator])-1].Status = status + } else { + res[validator] = append(res[validator], &types.ValidatorProposal{ + Index: validator, + Status: status, + Slot: proposalSlot, + }) + } + resMux.Unlock() + + } + return true + }, gcp_bigtable.RowFilter(filter)) + + return err + }) + } + + if err := g.Wait(); err != nil { return nil, err } @@ -1469,7 +1787,8 @@ func (bigtable *Bigtable) GetValidatorProposalHistory(validators []uint64, start } func (bigtable *Bigtable) SaveValidatorIncomeDetails(epoch uint64, rewards map[uint64]*itypes.ValidatorEpochIncome) error { - ctx, cancel := context.WithTimeout(context.Background(), time.Second*30) + + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) defer cancel() start := time.Now() @@ -1477,11 +1796,10 @@ func (bigtable *Bigtable) SaveValidatorIncomeDetails(epoch uint64, rewards map[u total := &itypes.ValidatorEpochIncome{} - mut := gcp_bigtable.NewMutation() + muts := make([]*gcp_bigtable.Mutation, 0, MAX_BATCH_MUTATIONS) + keys := make([]string, 0, MAX_BATCH_MUTATIONS) - muts := 0 for i, rewardDetails := range rewards { - muts++ data, err := proto.Marshal(rewardDetails) @@ -1489,137 +1807,91 @@ func (bigtable *Bigtable) SaveValidatorIncomeDetails(epoch uint64, rewards map[u return err } - mut.Set(INCOME_DETAILS_COLUMN_FAMILY, fmt.Sprintf("%d", i), ts, data) + mut := &gcp_bigtable.Mutation{} + mut.Set(INCOME_DETAILS_COLUMN_FAMILY, "i", ts, data) + key := fmt.Sprintf("%s:%s:%s:%s", bigtable.chainId, bigtable.validatorIndexToKey(i), INCOME_DETAILS_COLUMN_FAMILY, bigtable.reversedPaddedEpoch(epoch)) + + muts = append(muts, mut) + keys = append(keys, key) - if muts%100000 == 0 { - err := bigtable.tableBeaconchain.Apply(ctx, fmt.Sprintf("%s:e:b:%s", bigtable.chainId, reversedPaddedEpoch(epoch)), mut) + if len(muts) == MAX_BATCH_MUTATIONS { + errs, err := bigtable.tableValidatorsHistory.ApplyBulk(ctx, keys, muts) if err != nil { return err } - mut = gcp_bigtable.NewMutation() - } - - total.AttestationHeadReward += rewardDetails.AttestationHeadReward - total.AttestationSourceReward += rewardDetails.AttestationSourceReward - total.AttestationSourcePenalty += rewardDetails.AttestationSourcePenalty - total.AttestationTargetReward += rewardDetails.AttestationTargetReward - total.AttestationTargetPenalty += rewardDetails.AttestationTargetPenalty - total.FinalityDelayPenalty += rewardDetails.FinalityDelayPenalty - total.ProposerSlashingInclusionReward += rewardDetails.ProposerSlashingInclusionReward - total.ProposerAttestationInclusionReward += rewardDetails.ProposerAttestationInclusionReward - total.ProposerSyncInclusionReward += rewardDetails.ProposerSyncInclusionReward - total.SyncCommitteeReward += rewardDetails.SyncCommitteeReward - total.SyncCommitteePenalty += rewardDetails.SyncCommitteePenalty - total.SlashingReward += rewardDetails.SlashingReward - total.SlashingPenalty += rewardDetails.SlashingPenalty - total.TxFeeRewardWei = utils.AddBigInts(total.TxFeeRewardWei, rewardDetails.TxFeeRewardWei) - } - - sum, err := proto.Marshal(total) - if err != nil { - return err - } - - mut.Set(STATS_COLUMN_FAMILY, SUM_COLUMN, ts, sum) - - err = bigtable.tableBeaconchain.Apply(ctx, fmt.Sprintf("%s:e:b:%s", bigtable.chainId, reversedPaddedEpoch(epoch)), mut) - if err != nil { - return err - } - - logger.Infof("exported validator income details for epoch %v to bigtable in %v", epoch, time.Since(start)) - return nil -} - -func (bigtable *Bigtable) GetEpochIncomeHistoryDescending(startEpoch uint64, endEpoch uint64) (*itypes.ValidatorEpochIncome, error) { - ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(time.Second*30)) - defer cancel() - - ranges := bigtable.getEpochRanges(startEpoch, endEpoch) - family := gcp_bigtable.FamilyFilter(STATS_COLUMN_FAMILY) - columnFilter := gcp_bigtable.ColumnFilter(SUM_COLUMN) - filter := gcp_bigtable.RowFilter(gcp_bigtable.ChainFilters(family, columnFilter)) - - res := itypes.ValidatorEpochIncome{} - - err := bigtable.tableBeaconchain.ReadRows(ctx, ranges, func(r gcp_bigtable.Row) bool { - if len(r[STATS_COLUMN_FAMILY]) == 0 { - return false - } - err := proto.Unmarshal(r[STATS_COLUMN_FAMILY][0].Value, &res) - if err != nil { - logger.Errorf("error decoding income data for row %v: %v", r.Key(), err) - return false - } - return true - }, filter) - - if err != nil { - return nil, fmt.Errorf("error reading income statistics from bigtable for epoch: %v err: %w", startEpoch, err) - } - - return &res, nil -} - -func (bigtable *Bigtable) GetEpochIncomeHistory(epoch uint64) (*itypes.ValidatorEpochIncome, error) { - - ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(time.Second*30)) - defer cancel() - - key := fmt.Sprintf("%s:e:b:%s", bigtable.chainId, reversedPaddedEpoch(epoch)) - - family := gcp_bigtable.FamilyFilter(STATS_COLUMN_FAMILY) - columnFilter := gcp_bigtable.ColumnFilter(SUM_COLUMN) - filter := gcp_bigtable.RowFilter(gcp_bigtable.ChainFilters(family, columnFilter)) + for _, err := range errs { + return err + } + muts = make([]*gcp_bigtable.Mutation, 0, MAX_BATCH_MUTATIONS) + keys = make([]string, 0, MAX_BATCH_MUTATIONS) + } - row, err := bigtable.tableBeaconchain.ReadRow(ctx, key, filter) - if err != nil { - return nil, fmt.Errorf("error reading income statistics from bigtable for epoch: %v err: %w", epoch, err) + total.AttestationHeadReward += rewardDetails.AttestationHeadReward + total.AttestationSourceReward += rewardDetails.AttestationSourceReward + total.AttestationSourcePenalty += rewardDetails.AttestationSourcePenalty + total.AttestationTargetReward += rewardDetails.AttestationTargetReward + total.AttestationTargetPenalty += rewardDetails.AttestationTargetPenalty + total.FinalityDelayPenalty += rewardDetails.FinalityDelayPenalty + total.ProposerSlashingInclusionReward += rewardDetails.ProposerSlashingInclusionReward + total.ProposerAttestationInclusionReward += rewardDetails.ProposerAttestationInclusionReward + total.ProposerSyncInclusionReward += rewardDetails.ProposerSyncInclusionReward + total.SyncCommitteeReward += rewardDetails.SyncCommitteeReward + total.SyncCommitteePenalty += rewardDetails.SyncCommitteePenalty + total.SlashingReward += rewardDetails.SlashingReward + total.SlashingPenalty += rewardDetails.SlashingPenalty + total.TxFeeRewardWei = utils.AddBigInts(total.TxFeeRewardWei, rewardDetails.TxFeeRewardWei) } - if row != nil { - res := itypes.ValidatorEpochIncome{} - err := proto.Unmarshal(row[STATS_COLUMN_FAMILY][0].Value, &res) + if len(muts) > 0 { + errs, err := bigtable.tableValidatorsHistory.ApplyBulk(ctx, keys, muts) + if err != nil { - return nil, fmt.Errorf("error decoding income data for row %v: %w", row.Key(), err) + return err + } + for _, err := range errs { + return err } - return &res, nil } - // if there is no result we have to calculate the sum - income, err := bigtable.GetValidatorIncomeDetailsHistory([]uint64{}, epoch, 1) + sum, err := proto.Marshal(total) if err != nil { - logger.WithError(err).Error("error getting validator income history") + return err } - total := &itypes.ValidatorEpochIncome{} + mut := &gcp_bigtable.Mutation{} + mut.Set(STATS_COLUMN_FAMILY, SUM_COLUMN, ts, sum) - for _, epochs := range income { - for _, details := range epochs { - total.AttestationHeadReward += details.AttestationHeadReward - total.AttestationSourceReward += details.AttestationSourceReward - total.AttestationSourcePenalty += details.AttestationSourcePenalty - total.AttestationTargetReward += details.AttestationTargetReward - total.AttestationTargetPenalty += details.AttestationTargetPenalty - total.FinalityDelayPenalty += details.FinalityDelayPenalty - total.ProposerSlashingInclusionReward += details.ProposerSlashingInclusionReward - total.ProposerAttestationInclusionReward += details.ProposerAttestationInclusionReward - total.ProposerSyncInclusionReward += details.ProposerSyncInclusionReward - total.SyncCommitteeReward += details.SyncCommitteeReward - total.SyncCommitteePenalty += details.SyncCommitteePenalty - total.SlashingReward += details.SlashingReward - total.SlashingPenalty += details.SlashingPenalty - total.TxFeeRewardWei = utils.AddBigInts(total.TxFeeRewardWei, details.TxFeeRewardWei) - } + err = bigtable.tableValidatorsHistory.Apply(ctx, fmt.Sprintf("%s:%s:%s", bigtable.chainId, SUM_COLUMN, bigtable.reversedPaddedEpoch(epoch)), mut) + if err != nil { + return err } - return total, nil + logger.Infof("exported validator income details for epoch %v to bigtable in %v", epoch, time.Since(start)) + return nil } // GetValidatorIncomeDetailsHistory returns the validator income details // startEpoch & endEpoch are inclusive func (bigtable *Bigtable) GetValidatorIncomeDetailsHistory(validators []uint64, startEpoch uint64, endEpoch uint64) (map[uint64]map[uint64]*itypes.ValidatorEpochIncome, error) { + tmr := time.NewTimer(REPORT_TIMEOUT) + defer tmr.Stop() + go func() { + <-tmr.C + logger.WithFields(logrus.Fields{ + "validatorsCount": len(validators), + "startEpoch": startEpoch, + "endEpoch": endEpoch, + }).Warnf("%s call took longer than %v", utils.GetCurrentFuncName(), REPORT_TIMEOUT) + }() + + if len(validators) == 0 { + return nil, fmt.Errorf("passing empty validator array is unsupported") + } + + batchSize := 1000 + concurrency := 10 + if startEpoch > endEpoch { startEpoch = 0 } @@ -1627,89 +1899,81 @@ func (bigtable *Bigtable) GetValidatorIncomeDetailsHistory(validators []uint64, ctx, cancel := context.WithTimeout(context.Background(), time.Second*180) defer cancel() - ranges := bigtable.getEpochRanges(startEpoch, endEpoch) res := make(map[uint64]map[uint64]*itypes.ValidatorEpochIncome, len(validators)) + resMux := &sync.Mutex{} - valLen := len(validators) + filter := gcp_bigtable.LatestNFilter(1) - // read entire row if you require more than 1000 validators - var columnFilters []gcp_bigtable.Filter - if valLen < 1000 { - columnFilters = make([]gcp_bigtable.Filter, 0, valLen) - for _, validator := range validators { - columnFilters = append(columnFilters, gcp_bigtable.ColumnFilter(fmt.Sprintf("%d", validator))) + g, gCtx := errgroup.WithContext(ctx) + g.SetLimit(concurrency) + + for i := 0; i < len(validators); i += batchSize { + + upperBound := i + batchSize + if len(validators) < upperBound { + upperBound = len(validators) } - } + vals := validators[i:upperBound] - filter := gcp_bigtable.ChainFilters( - gcp_bigtable.FamilyFilter(INCOME_DETAILS_COLUMN_FAMILY), - gcp_bigtable.InterleaveFilters(columnFilters...), - gcp_bigtable.LatestNFilter(1), - ) + g.Go(func() error { + select { + case <-gCtx.Done(): + return gCtx.Err() + default: + } + ranges := bigtable.getValidatorsEpochRanges(vals, INCOME_DETAILS_COLUMN_FAMILY, startEpoch, endEpoch) + err := bigtable.tableValidatorsHistory.ReadRows(ctx, ranges, func(r gcp_bigtable.Row) bool { + keySplit := strings.Split(r.Key(), ":") - if len(columnFilters) == 1 { // special case to retrieve data for one validator - filter = gcp_bigtable.ChainFilters( - gcp_bigtable.FamilyFilter(INCOME_DETAILS_COLUMN_FAMILY), - columnFilters[0], - gcp_bigtable.LatestNFilter(1), - ) - } - if len(columnFilters) == 0 { // special case to retrieve data for all validators - filter = gcp_bigtable.ChainFilters( - gcp_bigtable.FamilyFilter(INCOME_DETAILS_COLUMN_FAMILY), - gcp_bigtable.LatestNFilter(1), - ) - } + validator, err := bigtable.validatorKeyToIndex(keySplit[1]) + if err != nil { + logger.Errorf("error parsing validator from row key %v: %v", r.Key(), err) + return false + } - err := bigtable.tableBeaconchain.ReadRows(ctx, ranges, func(r gcp_bigtable.Row) bool { - keySplit := strings.Split(r.Key(), ":") + epoch, err := strconv.ParseUint(keySplit[3], 10, 64) + if err != nil { + logger.Errorf("error parsing epoch from row key %v: %v", r.Key(), err) + return false + } - epoch, err := strconv.ParseUint(keySplit[3], 10, 64) - if err != nil { - logger.Errorf("error parsing epoch from row key %v: %v", r.Key(), err) - return false - } + for _, ri := range r[INCOME_DETAILS_COLUMN_FAMILY] { + incomeDetails := &itypes.ValidatorEpochIncome{} + err = proto.Unmarshal(ri.Value, incomeDetails) + if err != nil { + logger.Errorf("error decoding validator income data for row %v: %v", r.Key(), err) + return false + } - // logger.Info(max_epoch - epoch) - for _, ri := range r[INCOME_DETAILS_COLUMN_FAMILY] { - validator, err := strconv.ParseUint(strings.TrimPrefix(ri.Column, INCOME_DETAILS_COLUMN_FAMILY+":"), 10, 64) - if err != nil { - logger.Errorf("error parsing validator from column key %v: %v", ri.Column, err) - return false - } + resMux.Lock() + if res[validator] == nil { + res[validator] = make(map[uint64]*itypes.ValidatorEpochIncome) + } - incomeDetails := &itypes.ValidatorEpochIncome{} - err = proto.Unmarshal(ri.Value, incomeDetails) - if err != nil { - logger.Errorf("error decoding validator income data for row %v: %v", r.Key(), err) - return false - } + res[validator][MAX_EPOCH-epoch] = incomeDetails + resMux.Unlock() + } + return true + }, gcp_bigtable.RowFilter(filter)) - if res[validator] == nil { - res[validator] = make(map[uint64]*itypes.ValidatorEpochIncome) - } + return err + }) + } - res[validator][max_epoch-epoch] = incomeDetails - } - return true - }, gcp_bigtable.RowFilter(filter)) - if err != nil { + if err := g.Wait(); err != nil { return nil, err } return res, nil } -// GetValidatorIncomeDetailsHistory returns the validator income details +// GetAggregatedValidatorIncomeDetailsHistory returns aggregated validator income details // startEpoch & endEpoch are inclusive func (bigtable *Bigtable) GetAggregatedValidatorIncomeDetailsHistory(validators []uint64, startEpoch uint64, endEpoch uint64) (map[uint64]*itypes.ValidatorEpochIncome, error) { if startEpoch > endEpoch { startEpoch = 0 } - ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(time.Minute*10)) - defer cancel() - type ResultContainer struct { mu sync.Mutex res map[uint64]*itypes.ValidatorEpochIncome @@ -1717,100 +1981,47 @@ func (bigtable *Bigtable) GetAggregatedValidatorIncomeDetailsHistory(validators resultContainer := ResultContainer{} resultContainer.res = make(map[uint64]*itypes.ValidatorEpochIncome, len(validators)) - valLen := len(validators) + batchSize := 10000 + for i := 0; i < len(validators); i += batchSize { - // read entire row if you require more than 1000 validators - var columnFilters []gcp_bigtable.Filter - if valLen < 1000 { - columnFilters = make([]gcp_bigtable.Filter, 0, valLen) - for _, validator := range validators { - columnFilters = append(columnFilters, gcp_bigtable.ColumnFilter(fmt.Sprintf("%d", validator))) + upperBound := i + batchSize + if len(validators) < upperBound { + upperBound = len(validators) } - } + vals := validators[i:upperBound] - filter := gcp_bigtable.ChainFilters( - gcp_bigtable.FamilyFilter(INCOME_DETAILS_COLUMN_FAMILY), - gcp_bigtable.InterleaveFilters(columnFilters...), - gcp_bigtable.LatestNFilter(1), - ) + logrus.Infof("retrieving validator income stats for validators %v - %v", vals[0], vals[len(vals)-1]) - if len(columnFilters) == 1 { // special case to retrieve data for one validators - filter = gcp_bigtable.ChainFilters( - gcp_bigtable.FamilyFilter(INCOME_DETAILS_COLUMN_FAMILY), - columnFilters[0], - gcp_bigtable.LatestNFilter(1), - ) - } - if len(columnFilters) == 0 { // special case to retrieve data for all validators - filter = gcp_bigtable.ChainFilters( - gcp_bigtable.FamilyFilter(INCOME_DETAILS_COLUMN_FAMILY), - gcp_bigtable.LatestNFilter(1), - ) - } + res, err := bigtable.GetValidatorIncomeDetailsHistory(vals, startEpoch, endEpoch) - g, gCtx := errgroup.WithContext(ctx) - batchSize := utilMath.MaxU64(5, (endEpoch-startEpoch)/5) // we can speed up the loading by splitting it up. Making the batchSize even smaller has no effect but increases the memory consumption. - for e := startEpoch; e < endEpoch; e += batchSize { - fromEpoch := e - toEpoch := fromEpoch + batchSize - 1 - if toEpoch > endEpoch { - toEpoch = endEpoch + if err != nil { + return nil, err } + resultContainer.mu.Lock() + for validator, epochs := range res { + for _, rewardDetails := range epochs { - g.Go(func() error { - select { - case <-gCtx.Done(): - return gCtx.Err() - default: - } - ranges := bigtable.getEpochRanges(fromEpoch, toEpoch) - - err := bigtable.tableBeaconchain.ReadRows(ctx, ranges, func(r gcp_bigtable.Row) bool { - - resultContainer.mu.Lock() - - for _, ri := range r[INCOME_DETAILS_COLUMN_FAMILY] { - validator, err := strconv.ParseUint(strings.TrimPrefix(ri.Column, INCOME_DETAILS_COLUMN_FAMILY+":"), 10, 64) - if err != nil { - logger.Errorf("error parsing validator from column key %v: %v", ri.Column, err) - return false - } - - rewardDetails := &itypes.ValidatorEpochIncome{} - err = proto.Unmarshal(ri.Value, rewardDetails) - if err != nil { - logger.Errorf("error decoding validator income data for row %v: %v", r.Key(), err) - return false - } - - if resultContainer.res[validator] == nil { - resultContainer.res[validator] = &itypes.ValidatorEpochIncome{} - } - - resultContainer.res[validator].AttestationHeadReward += rewardDetails.AttestationHeadReward - resultContainer.res[validator].AttestationSourceReward += rewardDetails.AttestationSourceReward - resultContainer.res[validator].AttestationSourcePenalty += rewardDetails.AttestationSourcePenalty - resultContainer.res[validator].AttestationTargetReward += rewardDetails.AttestationTargetReward - resultContainer.res[validator].AttestationTargetPenalty += rewardDetails.AttestationTargetPenalty - resultContainer.res[validator].FinalityDelayPenalty += rewardDetails.FinalityDelayPenalty - resultContainer.res[validator].ProposerSlashingInclusionReward += rewardDetails.ProposerSlashingInclusionReward - resultContainer.res[validator].ProposerAttestationInclusionReward += rewardDetails.ProposerAttestationInclusionReward - resultContainer.res[validator].ProposerSyncInclusionReward += rewardDetails.ProposerSyncInclusionReward - resultContainer.res[validator].SyncCommitteeReward += rewardDetails.SyncCommitteeReward - resultContainer.res[validator].SyncCommitteePenalty += rewardDetails.SyncCommitteePenalty - resultContainer.res[validator].SlashingReward += rewardDetails.SlashingReward - resultContainer.res[validator].SlashingPenalty += rewardDetails.SlashingPenalty - resultContainer.res[validator].TxFeeRewardWei = utils.AddBigInts(resultContainer.res[validator].TxFeeRewardWei, rewardDetails.TxFeeRewardWei) + if resultContainer.res[validator] == nil { + resultContainer.res[validator] = &itypes.ValidatorEpochIncome{} } - resultContainer.mu.Unlock() - return true - }, gcp_bigtable.RowFilter(filter)) - return err - }) - } - if err := g.Wait(); err != nil { - return nil, err + resultContainer.res[validator].AttestationHeadReward += rewardDetails.AttestationHeadReward + resultContainer.res[validator].AttestationSourceReward += rewardDetails.AttestationSourceReward + resultContainer.res[validator].AttestationSourcePenalty += rewardDetails.AttestationSourcePenalty + resultContainer.res[validator].AttestationTargetReward += rewardDetails.AttestationTargetReward + resultContainer.res[validator].AttestationTargetPenalty += rewardDetails.AttestationTargetPenalty + resultContainer.res[validator].FinalityDelayPenalty += rewardDetails.FinalityDelayPenalty + resultContainer.res[validator].ProposerSlashingInclusionReward += rewardDetails.ProposerSlashingInclusionReward + resultContainer.res[validator].ProposerAttestationInclusionReward += rewardDetails.ProposerAttestationInclusionReward + resultContainer.res[validator].ProposerSyncInclusionReward += rewardDetails.ProposerSyncInclusionReward + resultContainer.res[validator].SyncCommitteeReward += rewardDetails.SyncCommitteeReward + resultContainer.res[validator].SyncCommitteePenalty += rewardDetails.SyncCommitteePenalty + resultContainer.res[validator].SlashingReward += rewardDetails.SlashingReward + resultContainer.res[validator].SlashingPenalty += rewardDetails.SlashingPenalty + resultContainer.res[validator].TxFeeRewardWei = utils.AddBigInts(resultContainer.res[validator].TxFeeRewardWei, rewardDetails.TxFeeRewardWei) + } + } + resultContainer.mu.Unlock() } return resultContainer.res, nil @@ -1818,100 +2029,92 @@ func (bigtable *Bigtable) GetAggregatedValidatorIncomeDetailsHistory(validators // Deletes all block data from bigtable func (bigtable *Bigtable) DeleteEpoch(epoch uint64) error { + // TOTO: Implement + return fmt.Errorf("NOT IMPLEMENTED") +} - // First receive all keys that were written by this block (entities & indices) - keys := make([]string, 0, 33) - startSlot := epoch * utils.Config.Chain.ClConfig.SlotsPerEpoch - endSlot := (epoch+1)*utils.Config.Chain.ClConfig.SlotsPerEpoch - 1 - - logger.Infof("deleting epoch %v (slot %v to %v)", epoch, startSlot, endSlot) - for slot := startSlot; slot <= endSlot; slot++ { - keys = append(keys, fmt.Sprintf("%s:e:%s:s:%s", bigtable.chainId, reversedPaddedEpoch(slot/utils.Config.Chain.ClConfig.SlotsPerEpoch), reversedPaddedSlot(slot))) +func (bigtable *Bigtable) getValidatorsEpochRanges(validatorIndices []uint64, prefix string, startEpoch uint64, endEpoch uint64) gcp_bigtable.RowRangeList { + if endEpoch > math.MaxInt64 { + endEpoch = 0 + } + if endEpoch < startEpoch { // handle overflows + startEpoch = 0 } - keys = append(keys, fmt.Sprintf("%s:e:b:%s", bigtable.chainId, reversedPaddedEpoch(epoch))) - // for _, k := range keys { - // logger.Info(k) - // } + ranges := make(gcp_bigtable.RowRangeList, 0, int((endEpoch-startEpoch+1))*len(validatorIndices)) - // Delete all of those keys - mutsDelete := &types.BulkMutations{ - Keys: make([]string, 0, len(keys)), - Muts: make([]*gcp_bigtable.Mutation, 0, len(keys)), - } - for _, key := range keys { - mutDelete := gcp_bigtable.NewMutation() - mutDelete.DeleteRow() - mutsDelete.Keys = append(mutsDelete.Keys, key) - mutsDelete.Muts = append(mutsDelete.Muts, mutDelete) - } + for _, validatorIndex := range validatorIndices { + validatorKey := bigtable.validatorIndexToKey(validatorIndex) - err := bigtable.WriteBulk(mutsDelete, bigtable.tableBeaconchain) - if err != nil { - return err + // epochs are sorted descending, so start with the largest epoch and end with the smallest + // add \x00 to make the range inclusive + rangeEnd := fmt.Sprintf("%s:%s:%s:%s%s", bigtable.chainId, validatorKey, prefix, bigtable.reversedPaddedEpoch(startEpoch), "\x00") + rangeStart := fmt.Sprintf("%s:%s:%s:%s", bigtable.chainId, validatorKey, prefix, bigtable.reversedPaddedEpoch(endEpoch)) + ranges = append(ranges, gcp_bigtable.NewRange(rangeStart, rangeEnd)) } - - return nil + return ranges } -func (bigtable *Bigtable) getSlotRanges(startEpoch uint64, endEpoch uint64) gcp_bigtable.RowRangeList { +func (bigtable *Bigtable) getValidatorsEpochSlotRanges(validatorIndices []uint64, prefix string, startEpoch uint64, endEpoch uint64) gcp_bigtable.RowRangeList { + if endEpoch > math.MaxInt64 { + endEpoch = 0 + } if endEpoch < startEpoch { // handle overflows startEpoch = 0 } - ranges := gcp_bigtable.RowRangeList{} - if startEpoch == 0 { // special case when the 0 epoch is included - rangeEnd := fmt.Sprintf("%s:e:%s:s:%s", bigtable.chainId, reversedPaddedEpoch(0), ":") - rangeStart := fmt.Sprintf("%s:e:%s:s:", bigtable.chainId, reversedPaddedEpoch(0)) - ranges = append(ranges, gcp_bigtable.NewRange(rangeStart, rangeEnd)) + ranges := make(gcp_bigtable.RowRangeList, 0, int((endEpoch-startEpoch+1))*len(validatorIndices)) - // epochs are sorted descending, so start with the larges epoch and end with the smallest - // add ':', a character lexicographically after digits, to make the range inclusive - if startEpoch < endEpoch { - rangeEnd = fmt.Sprintf("%s:e:%s:s:%s", bigtable.chainId, reversedPaddedEpoch(startEpoch+1), ":") - rangeStart = fmt.Sprintf("%s:e:%s:s:", bigtable.chainId, reversedPaddedEpoch(endEpoch)) - ranges = append(ranges, gcp_bigtable.NewRange(rangeStart, rangeEnd)) - } - } else { - // epochs are sorted descending, so start with the larges epoch and end with the smallest - // add ':', a character lexicographically after digits, to make the range inclusive - rangeEnd := fmt.Sprintf("%s:e:%s:s:%s", bigtable.chainId, reversedPaddedEpoch(startEpoch), ":") - rangeStart := fmt.Sprintf("%s:e:%s:s:", bigtable.chainId, reversedPaddedEpoch(endEpoch)) + for _, validatorIndex := range validatorIndices { + validatorKey := bigtable.validatorIndexToKey(validatorIndex) + + rangeEnd := fmt.Sprintf("%s:%s:%s:%s:%s%s", bigtable.chainId, validatorKey, prefix, bigtable.reversedPaddedEpoch(startEpoch), bigtable.reversedPaddedSlot(startEpoch*utils.Config.Chain.ClConfig.SlotsPerEpoch), "\x00") + rangeStart := fmt.Sprintf("%s:%s:%s:%s:%s", bigtable.chainId, validatorKey, prefix, bigtable.reversedPaddedEpoch(endEpoch), bigtable.reversedPaddedSlot(endEpoch*utils.Config.Chain.ClConfig.SlotsPerEpoch+utils.Config.Chain.ClConfig.SlotsPerEpoch-1)) ranges = append(ranges, gcp_bigtable.NewRange(rangeStart, rangeEnd)) + } return ranges } -func (bigtable *Bigtable) getEpochRanges(startEpoch uint64, endEpoch uint64) gcp_bigtable.RowRangeList { - - if endEpoch < startEpoch { // handle overflows - startEpoch = 0 +func (bigtable *Bigtable) getValidatorSlotRanges(validatorIndices []uint64, prefix string, startSlot uint64, endSlot uint64) gcp_bigtable.RowRangeList { + if endSlot > math.MaxInt64 { + endSlot = 0 + } + if endSlot < startSlot { // handle overflows + startSlot = 0 } - ranges := gcp_bigtable.RowRangeList{} - if startEpoch == 0 { // special case when the 0 epoch is included - rangeEnd := fmt.Sprintf("%s:e:b:%s%s", bigtable.chainId, reversedPaddedEpoch(0), "\x00") - rangeStart := fmt.Sprintf("%s:e:b:%s", bigtable.chainId, reversedPaddedEpoch(0)) - ranges = append(ranges, gcp_bigtable.NewRange(rangeStart, rangeEnd)) + startEpoch := utils.EpochOfSlot(startSlot) + endEpoch := utils.EpochOfSlot(endSlot) - // epochs are sorted descending, so start with the largest epoch and end with the smallest - // add \x00 to make the range inclusive - if startEpoch < endEpoch { - rangeEnd = fmt.Sprintf("%s:e:b:%s%s", bigtable.chainId, reversedPaddedEpoch(startEpoch+1), "\x00") - rangeStart = fmt.Sprintf("%s:e:b:%s", bigtable.chainId, reversedPaddedEpoch(endEpoch)) - ranges = append(ranges, gcp_bigtable.NewRange(rangeStart, rangeEnd)) - } - } else { - // epochs are sorted descending, so start with the largest epoch and end with the smallest - // add \x00 to make the range inclusive - rangeEnd := fmt.Sprintf("%s:e:b:%s%s", bigtable.chainId, reversedPaddedEpoch(startEpoch), "\x00") - rangeStart := fmt.Sprintf("%s:e:b:%s", bigtable.chainId, reversedPaddedEpoch(endEpoch)) + ranges := make(gcp_bigtable.RowRangeList, 0, len(validatorIndices)) + + for _, validatorIndex := range validatorIndices { + validatorKey := bigtable.validatorIndexToKey(validatorIndex) + + rangeEnd := fmt.Sprintf("%s:%s:%s:%s:%s%s", bigtable.chainId, validatorKey, prefix, bigtable.reversedPaddedEpoch(startEpoch), bigtable.reversedPaddedSlot(startSlot), "\x00") + rangeStart := fmt.Sprintf("%s:%s:%s:%s:%s", bigtable.chainId, validatorKey, prefix, bigtable.reversedPaddedEpoch(endEpoch), bigtable.reversedPaddedSlot(endSlot)) ranges = append(ranges, gcp_bigtable.NewRange(rangeStart, rangeEnd)) + } return ranges } +func (bigtable *Bigtable) validatorIndexToKey(index uint64) string { + return utils.ReverseString(fmt.Sprintf("%d", index)) +} + +func (bigtable *Bigtable) validatorKeyToIndex(key string) (uint64, error) { + key = utils.ReverseString(key) + indexKey, err := strconv.ParseUint(key, 10, 64) + + if err != nil { + return 0, err + } + return indexKey, nil +} + func (bigtable *Bigtable) ClearByPrefix(family, prefix string, dryRun bool) ([]string, error) { if family == "" || prefix == "" { return []string{}, fmt.Errorf("please provide family [%v] and prefix [%v]", family, prefix) @@ -1954,12 +2157,11 @@ func (bigtable *Bigtable) ClearByPrefix(family, prefix string, dryRun bool) ([]s return deleteKeys, err } -func GetCurrentDayClIncome(validator_indices []uint64) (map[uint64]int64, map[uint64]int64, error) { +func GetCurrentDayClIncome(validator_indices []uint64) (map[uint64]int64, error) { dayIncome := make(map[uint64]int64) - dayProposerIncome := make(map[uint64]int64) lastDay, err := GetLastExportedStatisticDay() if err != nil { - return dayIncome, dayProposerIncome, err + return dayIncome, err } currentDay := uint64(lastDay + 1) @@ -1967,7 +2169,7 @@ func GetCurrentDayClIncome(validator_indices []uint64) (map[uint64]int64, map[ui endEpoch := startEpoch + utils.EpochsPerDay() - 1 income, err := BigtableClient.GetValidatorIncomeDetailsHistory(validator_indices, startEpoch, endEpoch) if err != nil { - return dayIncome, dayProposerIncome, err + return dayIncome, err } // agregate all epoch income data to total day income for each validator @@ -1977,37 +2179,110 @@ func GetCurrentDayClIncome(validator_indices []uint64) (map[uint64]int64, map[ui } for _, validatorEpochIncome := range validatorIncome { dayIncome[validatorIndex] += validatorEpochIncome.TotalClRewards() - dayProposerIncome[validatorIndex] += int64(validatorEpochIncome.ProposerAttestationInclusionReward) + int64(validatorEpochIncome.ProposerSlashingInclusionReward) + int64(validatorEpochIncome.ProposerSyncInclusionReward) } } - return dayIncome, dayProposerIncome, nil + return dayIncome, nil } -func GetCurrentDayProposerIncomeTotal(validator_indices []uint64) (int64, error) { - _, proposerIncome, err := GetCurrentDayClIncome(validator_indices) +func (bigtable *Bigtable) reversePaddedUserID(userID uint64) string { + return fmt.Sprintf("%09d", ^uint64(0)-userID) +} - if err != nil { - return 0, err - } +func (bigtable *Bigtable) reversedPaddedEpoch(epoch uint64) string { + return fmt.Sprintf("%09d", MAX_EPOCH-epoch) +} - proposerTotal := int64(0) +func (bigtable *Bigtable) reversedPaddedSlot(slot uint64) string { + return fmt.Sprintf("%09d", MAX_CL_BLOCK_NUMBER-slot) +} - for _, i := range proposerIncome { - proposerTotal += i +func (bigtable *Bigtable) MigrateIncomeDataV1V2Schema(epoch uint64) error { + type validatorEpochData struct { + ValidatorIndex uint64 + IncomeDetails *itypes.ValidatorEpochIncome } - return proposerTotal, nil -} + epochData := make(map[uint64]*validatorEpochData) + filter := gcp_bigtable.ChainFilters(gcp_bigtable.FamilyFilter(INCOME_DETAILS_COLUMN_FAMILY), gcp_bigtable.LatestNFilter(1)) + ctx := context.Background() -func reversePaddedUserID(userID uint64) string { - return fmt.Sprintf("%09d", ^uint64(0)-userID) -} + prefixEpochRange := gcp_bigtable.PrefixRange(fmt.Sprintf("%s:e:b:%s", bigtable.chainId, fmt.Sprintf("%09d", (MAX_EPOCH)-epoch))) -func reversedPaddedEpoch(epoch uint64) string { - return fmt.Sprintf("%09d", max_block_number-epoch) -} + err := bigtable.tableBeaconchain.ReadRows(ctx, prefixEpochRange, func(r gcp_bigtable.Row) bool { + // logger.Infof("processing row %v", r.Key()) + + keySplit := strings.Split(r.Key(), ":") + + rowKeyEpoch, err := strconv.ParseUint(keySplit[3], 10, 64) + if err != nil { + logger.Errorf("error parsing epoch from row key %v: %v", r.Key(), err) + return false + } + + rowKeyEpoch = MAX_EPOCH - rowKeyEpoch + + if epoch != rowKeyEpoch { + logger.Errorf("retrieved different epoch than requested, requested: %d, retrieved: %d", epoch, rowKeyEpoch) + } + + // logger.Infof("epoch is %d", rowKeyEpoch) + + for columnFamily, readItems := range r { + + for _, ri := range readItems { + + if ri.Column == "stats:sum" { // skip migrating the total epoch income stats + continue + } + + validator, err := strconv.ParseUint(strings.TrimPrefix(ri.Column, columnFamily+":"), 10, 64) + if err != nil { + logger.Errorf("error parsing validator from column key %v: %v", ri.Column, err) + return false + } + + // logger.Infof("retrieved field %s from column family %s for validator %d", ri.Column, columnFamily, validator) + if columnFamily == INCOME_DETAILS_COLUMN_FAMILY { + if epochData[validator] == nil { + epochData[validator] = &validatorEpochData{ + ValidatorIndex: validator, + } + } + // logger.Infof("processing income details data for validator %d", validator) + incomeDetails := &itypes.ValidatorEpochIncome{} + err = proto.Unmarshal(ri.Value, incomeDetails) + if err != nil { + logger.Errorf("error decoding validator income data for row %v: %v", r.Key(), err) + return false + } + + epochData[validator].IncomeDetails = incomeDetails + } else { + logger.Errorf("retrieved unexpected column family %s", columnFamily) + } + } + } -func reversedPaddedSlot(slot uint64) string { - return fmt.Sprintf("%09d", max_block_number-slot) + return true + }, gcp_bigtable.RowFilter(filter)) + + if err != nil { + return err + } + + incomeData := make(map[uint64]*itypes.ValidatorEpochIncome) + for _, validator := range epochData { + if validator.IncomeDetails == nil { + continue + } + incomeData[validator.ValidatorIndex] = validator.IncomeDetails + } + + err = bigtable.SaveValidatorIncomeDetails(epoch, incomeData) + if err != nil { + return err + } + + return nil } diff --git a/db/bigtable_admin.go b/db/bigtable_admin.go deleted file mode 100644 index 338139f331..0000000000 --- a/db/bigtable_admin.go +++ /dev/null @@ -1,130 +0,0 @@ -package db - -import ( - "context" - "eth2-exporter/cache" - "eth2-exporter/utils" - "log" - "time" - - gcp_bigtable "cloud.google.com/go/bigtable" -) - -type BigtableAdmin struct { - client *gcp_bigtable.AdminClient -} - -type CreateTables struct { - Name string - ColFams []CreateFamily -} - -type CreateFamily struct { - Name string - Policy gcp_bigtable.GCPolicy -} - -var CacheTable CreateTables = CreateTables{ - cache.TABLE_CACHE, - []CreateFamily{ - { - Name: cache.FAMILY_TEN_MINUTES, - Policy: gcp_bigtable.IntersectionPolicy(gcp_bigtable.MaxVersionsPolicy(1), gcp_bigtable.MaxAgePolicy(time.Minute*10)), - }, - { - Name: cache.FAMILY_ONE_HOUR, - Policy: gcp_bigtable.IntersectionPolicy(gcp_bigtable.MaxVersionsPolicy(1), gcp_bigtable.MaxAgePolicy(time.Hour)), - }, - { - Name: cache.FAMILY_ONE_DAY, - Policy: gcp_bigtable.IntersectionPolicy(gcp_bigtable.MaxVersionsPolicy(1), gcp_bigtable.MaxAgePolicy(time.Hour*24)), - }, - }, -} - -var BigAdminClient *BigtableAdmin - -func MustInitBigtableAdmin(ctx context.Context, project, instance string) { - admin, err := gcp_bigtable.NewAdminClient(ctx, project, instance) - if err != nil { - log.Fatalf("Could not create admin client: %v", err) - } - - bta := &BigtableAdmin{ - client: admin, - } - - BigAdminClient = bta -} - -func (admin *BigtableAdmin) SetupBigtableCache() error { - - if err := admin.createTables([]CreateTables{CacheTable}); err != nil { - log.Fatal("Error occurred trying to create tables", err) - } - ctx, done := context.WithTimeout(context.Background(), time.Second*30) - defer done() - - for _, cf := range CacheTable.ColFams { - if err := admin.client.SetGCPolicy(ctx, CacheTable.Name, cf.Name, cf.Policy); err != nil { - return err - } - } - - return nil -} - -func (admin *BigtableAdmin) TearDownCache() error { - if err := admin.deleteTables([]CreateTables{CacheTable}); err != nil { - return err - } - return nil -} - -func (admin *BigtableAdmin) createTables(tables []CreateTables) error { - ctx := context.Background() - - tableList, err := admin.client.Tables(ctx) - if err != nil { - log.Printf("Could not fetch table list") - return err - } - - for _, table := range tables { - if !utils.SliceContains(tableList, table.Name) { - log.Printf("Creating table %s", table) - if err := admin.client.CreateTable(ctx, table.Name); err != nil { - log.Printf("Could not create table %s", table.Name) - return err - } - } - - tblInfo, err := admin.client.TableInfo(ctx, table.Name) - if err != nil { - log.Printf("Could not read info for table %s", table.Name) - return err - } - for _, colfam := range table.ColFams { - if !utils.SliceContains(tblInfo.Families, colfam.Name) { - if err := admin.client.CreateColumnFamily(ctx, table.Name, colfam.Name); err != nil { - log.Printf("Could not create column family %s: %v", colfam.Name, err) - return err - } - } - } - } - return nil -} - -func (admin *BigtableAdmin) deleteTables(tables []CreateTables) error { - ctx := context.Background() - for _, table := range tables { - if err := admin.client.DeleteTable(ctx, table.Name); err != nil { - log.Printf("Could not delete table %s err %s", table, err) - return err - } else { - log.Printf("Deleted Table: %v", table.Name) - } - } - return nil -} diff --git a/db/bigtable_eth1.go b/db/bigtable_eth1.go index 809e959734..54faffa900 100644 --- a/db/bigtable_eth1.go +++ b/db/bigtable_eth1.go @@ -112,7 +112,7 @@ func (bigtable *Bigtable) GetMetadatTable() *gcp_bigtable.Table { } func (bigtable *Bigtable) SaveBlock(block *types.Eth1Block) error { - ctx, cancel := context.WithTimeout(context.Background(), time.Second*10) + ctx, cancel := context.WithTimeout(context.Background(), time.Second*30) defer cancel() encodedBc, err := proto.Marshal(block) @@ -134,6 +134,14 @@ func (bigtable *Bigtable) SaveBlock(block *types.Eth1Block) error { } func (bigtable *Bigtable) GetBlockFromBlocksTable(number uint64) (*types.Eth1Block, error) { + tmr := time.NewTimer(REPORT_TIMEOUT) + defer tmr.Stop() + go func() { + <-tmr.C + logger.WithFields(logrus.Fields{ + "validators": number, + }).Warnf("%s call took longer than %v", utils.GetCurrentFuncName(), REPORT_TIMEOUT) + }() ctx, cancel := context.WithTimeout(context.Background(), time.Second*30) defer cancel() @@ -176,7 +184,7 @@ func (bigtable *Bigtable) CheckForGapsInBlocksTable(lookback int) (gapFound bool logger.Errorf("error parsing block number from key %v: %v", r.Key(), err) return false } - c = max_block_number - c + c = MAX_EL_BLOCK_NUMBER - c if c%10000 == 0 { logger.Infof("scanning, currently at block %v", c) @@ -200,6 +208,13 @@ func (bigtable *Bigtable) CheckForGapsInBlocksTable(lookback int) (gapFound bool } func (bigtable *Bigtable) GetLastBlockInBlocksTable() (int, error) { + tmr := time.NewTimer(REPORT_TIMEOUT) + defer tmr.Stop() + go func() { + <-tmr.C + logger.Warnf("%s call took longer than %v", utils.GetCurrentFuncName(), REPORT_TIMEOUT) + }() + ctx, cancel := context.WithTimeout(context.Background(), time.Second*30) defer cancel() redisKey := bigtable.chainId + ":lastBlockInBlocksTable" @@ -251,7 +266,7 @@ func (bigtable *Bigtable) CheckForGapsInDataTable(lookback int) error { logger.Errorf("error parsing block number from key %v: %v", r.Key(), err) return false } - c = max_block_number - c + c = MAX_EL_BLOCK_NUMBER - c if c%10000 == 0 { logger.Infof("scanning, currently at block %v", c) @@ -275,6 +290,13 @@ func (bigtable *Bigtable) CheckForGapsInDataTable(lookback int) error { } func (bigtable *Bigtable) GetLastBlockInDataTable() (int, error) { + tmr := time.NewTimer(REPORT_TIMEOUT) + defer tmr.Stop() + go func() { + <-tmr.C + logger.Warnf("%s call took longer than %v", utils.GetCurrentFuncName(), REPORT_TIMEOUT) + }() + ctx, cancel := context.WithTimeout(context.Background(), time.Second*30) defer cancel() redisKey := bigtable.chainId + ":lastBlockInDataTable" @@ -316,7 +338,7 @@ func (bigtable *Bigtable) getLastBlockInDataTableFromBigtable() (int, error) { logger.Errorf("error parsing block number from key %v: %v", r.Key(), err) return false } - c = max_block_number - c + c = MAX_EL_BLOCK_NUMBER - c lastBlock = c return c == 0 // required as the block with number 0 will be returned as first block before the most recent one @@ -343,7 +365,7 @@ func (bigtable *Bigtable) getLastBlockInBlocksTableFromBigtable() (int, error) { logger.Errorf("error parsing block number from key %v: %v", r.Key(), err) return false } - c = max_block_number - c + c = MAX_EL_BLOCK_NUMBER - c lastBlock = c return c == 0 // required as the block with number 0 will be returned as first block before the most recent one @@ -365,14 +387,19 @@ func (bigtable *Bigtable) SetLastBlockInDataTable(lastBlock int64) error { } func (bigtable *Bigtable) GetMostRecentBlockFromDataTable() (*types.Eth1BlockIndexed, error) { + tmr := time.NewTimer(REPORT_TIMEOUT) + defer tmr.Stop() + go func() { + <-tmr.C + logger.Warnf("%s call took longer than %v", utils.GetCurrentFuncName(), REPORT_TIMEOUT) + }() + ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(time.Second*30)) defer cancel() prefix := fmt.Sprintf("%s:B:", bigtable.chainId) rowRange := gcp_bigtable.PrefixRange(prefix) - rowFilter := gcp_bigtable.RowFilter(gcp_bigtable.ColumnFilter("d")) - block := types.Eth1BlockIndexed{} rowHandler := func(row gcp_bigtable.Row) bool { @@ -382,7 +409,7 @@ func (bigtable *Bigtable) GetMostRecentBlockFromDataTable() (*types.Eth1BlockInd return false } - c = max_block_number - c + c = MAX_EL_BLOCK_NUMBER - c err = proto.Unmarshal(row[DEFAULT_FAMILY][0].Value, &block) if err != nil { @@ -392,7 +419,7 @@ func (bigtable *Bigtable) GetMostRecentBlockFromDataTable() (*types.Eth1BlockInd return c == 0 } - err := bigtable.tableData.ReadRows(ctx, rowRange, rowHandler, rowFilter) + err := bigtable.tableData.ReadRows(ctx, rowRange, rowHandler, gcp_bigtable.LimitRows(2), gcp_bigtable.RowFilter(gcp_bigtable.ColumnFilter("d"))) if err != nil { return nil, err } @@ -424,6 +451,16 @@ func getBlockHandler(blocks *[]*types.Eth1BlockIndexed) func(gcp_bigtable.Row) b // high: highest (max) block number // low: lowest (min) block number func (bigtable *Bigtable) GetFullBlocksDescending(stream chan<- *types.Eth1Block, high, low uint64) error { + tmr := time.NewTimer(REPORT_TIMEOUT) + defer tmr.Stop() + go func() { + <-tmr.C + logger.WithFields(logrus.Fields{ + "high": high, + "low": low, + }).Warnf("%s call took longer than %v", utils.GetCurrentFuncName(), REPORT_TIMEOUT) + }() + ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(time.Second*180)) defer cancel() @@ -441,6 +478,7 @@ func (bigtable *Bigtable) GetFullBlocksDescending(stream chan<- *types.Eth1Block highKey := fmt.Sprintf("%s:%s", bigtable.chainId, reversedPaddedBlockNumber(high)) lowKey := fmt.Sprintf("%s:%s\x00", bigtable.chainId, reversedPaddedBlockNumber(low)) // add \x00 to make the range inclusive + limit := high - low + 1 // the low key will have a higher reverse padded number rowRange := gcp_bigtable.NewRange(highKey, lowKey) @@ -457,7 +495,7 @@ func (bigtable *Bigtable) GetFullBlocksDescending(stream chan<- *types.Eth1Block return true } - err := bigtable.tableBlocks.ReadRows(ctx, rowRange, rowHandler, rowFilter) + err := bigtable.tableBlocks.ReadRows(ctx, rowRange, rowHandler, rowFilter, gcp_bigtable.LimitRows(int64(limit))) if err != nil { return err } @@ -475,6 +513,16 @@ func (bigtable *Bigtable) GetFullBlocksDescending(stream chan<- *types.Eth1Block } func (bigtable *Bigtable) GetBlocksIndexedMultiple(blockNumbers []uint64, limit uint64) ([]*types.Eth1BlockIndexed, error) { + tmr := time.NewTimer(REPORT_TIMEOUT) + defer tmr.Stop() + go func() { + <-tmr.C + logger.WithFields(logrus.Fields{ + "blockNumbers": blockNumbers, + "limit": limit, + }).Warnf("%s call took longer than %v", utils.GetCurrentFuncName(), REPORT_TIMEOUT) + }() + rowList := gcp_bigtable.RowList{} for _, block := range blockNumbers { rowList = append(rowList, fmt.Sprintf("%s:B:%s", bigtable.chainId, reversedPaddedBlockNumber(block))) @@ -501,6 +549,16 @@ func (bigtable *Bigtable) GetBlocksIndexedMultiple(blockNumbers []uint64, limit // GetBlocksDescending gets blocks starting at block start func (bigtable *Bigtable) GetBlocksDescending(start, limit uint64) ([]*types.Eth1BlockIndexed, error) { + tmr := time.NewTimer(REPORT_TIMEOUT) + defer tmr.Stop() + go func() { + <-tmr.C + logger.WithFields(logrus.Fields{ + "start": start, + "limit": limit, + }).Warnf("%s call took longer than %v", utils.GetCurrentFuncName(), REPORT_TIMEOUT) + }() + if start < 1 || limit < 1 || limit > start { return nil, fmt.Errorf("invalid block range provided (start: %v, limit: %v)", start, limit) } @@ -539,7 +597,7 @@ func (bigtable *Bigtable) GetBlocksDescending(start, limit uint64) ([]*types.Eth } func reversedPaddedBlockNumber(blockNumber uint64) string { - return fmt.Sprintf("%09d", max_block_number-blockNumber) + return fmt.Sprintf("%09d", MAX_EL_BLOCK_NUMBER-blockNumber) } func reversePaddedBigtableTimestamp(timestamp *timestamppb.Timestamp) string { @@ -716,6 +774,7 @@ func (bigtable *Bigtable) IndexEventsWithTransformers(start, end int64, transfor close(stream) }(blocksChan) subG := new(errgroup.Group) + subG.SetLimit(int(concurrency)) for b := range blocksChan { block := b subG.Go(func() error { @@ -1911,6 +1970,16 @@ func (bigtable *Bigtable) TransformWithdrawals(block *types.Eth1Block, cache *fr } func (bigtable *Bigtable) GetEth1TxForAddress(prefix string, limit int64) ([]*types.Eth1TransactionIndexed, string, error) { + tmr := time.NewTimer(REPORT_TIMEOUT) + defer tmr.Stop() + go func() { + <-tmr.C + logger.WithFields(logrus.Fields{ + "prefix": prefix, + "limit": limit, + }).Warnf("%s call took longer than %v", utils.GetCurrentFuncName(), REPORT_TIMEOUT) + }() + ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(time.Second*30)) defer cancel() @@ -1960,6 +2029,16 @@ func (bigtable *Bigtable) GetEth1TxForAddress(prefix string, limit int64) ([]*ty } func (bigtable *Bigtable) GetAddressesNamesArMetadata(names *map[string]string, inputMetadata *map[string]*types.ERC20Metadata) (map[string]string, map[string]*types.ERC20Metadata, error) { + tmr := time.NewTimer(REPORT_TIMEOUT) + defer tmr.Stop() + go func() { + <-tmr.C + logger.WithFields(logrus.Fields{ + "names": names, + "inputMetadata": inputMetadata, + }).Warnf("%s call took longer than %v", utils.GetCurrentFuncName(), REPORT_TIMEOUT) + }() + outputMetadata := make(map[string]*types.ERC20Metadata) g := new(errgroup.Group) @@ -2001,6 +2080,15 @@ func (bigtable *Bigtable) GetAddressesNamesArMetadata(names *map[string]string, } func (bigtable *Bigtable) GetIndexedEth1Transaction(txHash []byte) (*types.Eth1TransactionIndexed, error) { + tmr := time.NewTimer(REPORT_TIMEOUT) + defer tmr.Stop() + go func() { + <-tmr.C + logger.WithFields(logrus.Fields{ + "txHash": txHash, + }).Warnf("%s call took longer than %v", utils.GetCurrentFuncName(), REPORT_TIMEOUT) + }() + ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(time.Second*30)) defer cancel() key := fmt.Sprintf("%s:TX:%x", bigtable.chainId, txHash) @@ -2023,6 +2111,17 @@ func (bigtable *Bigtable) GetIndexedEth1Transaction(txHash []byte) (*types.Eth1T } func (bigtable *Bigtable) GetAddressTransactionsTableData(address []byte, search string, pageToken string) (*types.DataTableResponse, error) { + tmr := time.NewTimer(REPORT_TIMEOUT) + defer tmr.Stop() + go func() { + <-tmr.C + logger.WithFields(logrus.Fields{ + "address": address, + "search": search, + "pageToken": pageToken, + }).Warnf("%s call took longer than %v", utils.GetCurrentFuncName(), REPORT_TIMEOUT) + }() + if pageToken == "" { pageToken = fmt.Sprintf("%s:I:TX:%x:%s:", bigtable.chainId, address, FILTER_TIME) } @@ -2074,6 +2173,16 @@ func (bigtable *Bigtable) GetAddressTransactionsTableData(address []byte, search } func (bigtable *Bigtable) GetEth1BlocksForAddress(prefix string, limit int64) ([]*types.Eth1BlockIndexed, string, error) { + tmr := time.NewTimer(REPORT_TIMEOUT) + defer tmr.Stop() + go func() { + <-tmr.C + logger.WithFields(logrus.Fields{ + "prefix": prefix, + "limit": limit, + }).Warnf("%s call took longer than %v", utils.GetCurrentFuncName(), REPORT_TIMEOUT) + }() + ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(time.Second*30)) defer cancel() @@ -2123,6 +2232,17 @@ func (bigtable *Bigtable) GetEth1BlocksForAddress(prefix string, limit int64) ([ } func (bigtable *Bigtable) GetAddressBlocksMinedTableData(address string, search string, pageToken string) (*types.DataTableResponse, error) { + tmr := time.NewTimer(REPORT_TIMEOUT) + defer tmr.Stop() + go func() { + <-tmr.C + logger.WithFields(logrus.Fields{ + "address": address, + "search": search, + "pageToken": pageToken, + }).Warnf("%s call took longer than %v", utils.GetCurrentFuncName(), REPORT_TIMEOUT) + }() + if pageToken == "" { pageToken = fmt.Sprintf("%s:I:B:%s:", bigtable.chainId, address) } @@ -2153,6 +2273,16 @@ func (bigtable *Bigtable) GetAddressBlocksMinedTableData(address string, search } func (bigtable *Bigtable) GetEth1UnclesForAddress(prefix string, limit int64) ([]*types.Eth1UncleIndexed, string, error) { + tmr := time.NewTimer(REPORT_TIMEOUT) + defer tmr.Stop() + go func() { + <-tmr.C + logger.WithFields(logrus.Fields{ + "prefix": prefix, + "limit": limit, + }).Warnf("%s call took longer than %v", utils.GetCurrentFuncName(), REPORT_TIMEOUT) + }() + ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(time.Second*30)) defer cancel() @@ -2202,6 +2332,17 @@ func (bigtable *Bigtable) GetEth1UnclesForAddress(prefix string, limit int64) ([ } func (bigtable *Bigtable) GetAddressUnclesMinedTableData(address string, search string, pageToken string) (*types.DataTableResponse, error) { + tmr := time.NewTimer(REPORT_TIMEOUT) + defer tmr.Stop() + go func() { + <-tmr.C + logger.WithFields(logrus.Fields{ + "address": address, + "search": search, + "pageToken": pageToken, + }).Warnf("%s call took longer than %v", utils.GetCurrentFuncName(), REPORT_TIMEOUT) + }() + if pageToken == "" { pageToken = fmt.Sprintf("%s:I:U:%s:", bigtable.chainId, address) } @@ -2335,6 +2476,16 @@ func (bigtable *Bigtable) GetAddressBlobTableData(address []byte, search string, } func (bigtable *Bigtable) GetEth1ItxForAddress(prefix string, limit int64) ([]*types.Eth1InternalTransactionIndexed, string, error) { + tmr := time.NewTimer(REPORT_TIMEOUT) + defer tmr.Stop() + go func() { + <-tmr.C + logger.WithFields(logrus.Fields{ + "prefix": prefix, + "limit": limit, + }).Warnf("%s call took longer than %v", utils.GetCurrentFuncName(), REPORT_TIMEOUT) + }() + ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(time.Second*30)) defer cancel() @@ -2388,6 +2539,17 @@ func (bigtable *Bigtable) GetEth1ItxForAddress(prefix string, limit int64) ([]*t } func (bigtable *Bigtable) GetAddressInternalTableData(address []byte, search string, pageToken string) (*types.DataTableResponse, error) { + tmr := time.NewTimer(REPORT_TIMEOUT) + defer tmr.Stop() + go func() { + <-tmr.C + logger.WithFields(logrus.Fields{ + "address": address, + "search": search, + "pageToken": pageToken, + }).Warnf("%s call took longer than %v", utils.GetCurrentFuncName(), REPORT_TIMEOUT) + }() + // defaults to most recent if pageToken == "" { pageToken = fmt.Sprintf("%s:I:ITX:%x:%s:", bigtable.chainId, address, FILTER_TIME) @@ -2437,6 +2599,16 @@ func (bigtable *Bigtable) GetAddressInternalTableData(address []byte, search str } func (bigtable *Bigtable) GetInternalTransfersForTransaction(transaction []byte, from []byte) ([]types.Transfer, error) { + tmr := time.NewTimer(REPORT_TIMEOUT) + defer tmr.Stop() + go func() { + <-tmr.C + logger.WithFields(logrus.Fields{ + "transaction": transaction, + "from": from, + }).Warnf("%s call took longer than %v", utils.GetCurrentFuncName(), REPORT_TIMEOUT) + }() + ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(time.Second*30)) defer cancel() @@ -2513,6 +2685,15 @@ func (bigtable *Bigtable) GetInternalTransfersForTransaction(transaction []byte, // currently only erc20 func (bigtable *Bigtable) GetArbitraryTokenTransfersForTransaction(transaction []byte) ([]*types.Transfer, error) { + tmr := time.NewTimer(REPORT_TIMEOUT) + defer tmr.Stop() + go func() { + <-tmr.C + logger.WithFields(logrus.Fields{ + "transaction": transaction, + }).Warnf("%s call took longer than %v", utils.GetCurrentFuncName(), REPORT_TIMEOUT) + }() + ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(time.Second*30)) defer cancel() // uses a more standard transfer in-between type so multiple token types can be handle before the final table response is generated @@ -2622,6 +2803,16 @@ func (bigtable *Bigtable) GetArbitraryTokenTransfersForTransaction(transaction [ } func (bigtable *Bigtable) GetEth1ERC20ForAddress(prefix string, limit int64) ([]*types.Eth1ERC20Indexed, string, error) { + tmr := time.NewTimer(REPORT_TIMEOUT) + defer tmr.Stop() + go func() { + <-tmr.C + logger.WithFields(logrus.Fields{ + "prefix": prefix, + "limit": limit, + }).Warnf("%s call took longer than %v", utils.GetCurrentFuncName(), REPORT_TIMEOUT) + }() + ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(time.Second*30)) defer cancel() @@ -2669,6 +2860,16 @@ func (bigtable *Bigtable) GetEth1ERC20ForAddress(prefix string, limit int64) ([] } func (bigtable *Bigtable) GetAddressErc20TableData(address []byte, search string, pageToken string) (*types.DataTableResponse, error) { + tmr := time.NewTimer(REPORT_TIMEOUT) + defer tmr.Stop() + go func() { + <-tmr.C + logger.WithFields(logrus.Fields{ + "address": address, + "search": search, + "pageToken": pageToken, + }).Warnf("%s call took longer than %v", utils.GetCurrentFuncName(), REPORT_TIMEOUT) + }() if pageToken == "" { pageToken = fmt.Sprintf("%s:I:ERC20:%x:%s:", bigtable.chainId, address, FILTER_TIME) @@ -2728,6 +2929,16 @@ func (bigtable *Bigtable) GetAddressErc20TableData(address []byte, search string } func (bigtable *Bigtable) GetEth1ERC721ForAddress(prefix string, limit int64) ([]*types.Eth1ERC721Indexed, string, error) { + tmr := time.NewTimer(REPORT_TIMEOUT) + defer tmr.Stop() + go func() { + <-tmr.C + logger.WithFields(logrus.Fields{ + "prefix": prefix, + "limit": limit, + }).Warnf("%s call took longer than %v", utils.GetCurrentFuncName(), REPORT_TIMEOUT) + }() + ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(time.Second*30)) defer cancel() @@ -2779,6 +2990,16 @@ func (bigtable *Bigtable) GetEth1ERC721ForAddress(prefix string, limit int64) ([ } func (bigtable *Bigtable) GetAddressErc721TableData(address string, search string, pageToken string) (*types.DataTableResponse, error) { + tmr := time.NewTimer(REPORT_TIMEOUT) + defer tmr.Stop() + go func() { + <-tmr.C + logger.WithFields(logrus.Fields{ + "address": address, + "search": search, + "pageToken": pageToken, + }).Warnf("%s call took longer than %v", utils.GetCurrentFuncName(), REPORT_TIMEOUT) + }() if pageToken == "" { pageToken = fmt.Sprintf("%s:I:ERC721:%s:%s:", bigtable.chainId, address, FILTER_TIME) @@ -2826,6 +3047,16 @@ func (bigtable *Bigtable) GetAddressErc721TableData(address string, search strin } func (bigtable *Bigtable) GetEth1ERC1155ForAddress(prefix string, limit int64) ([]*types.ETh1ERC1155Indexed, string, error) { + tmr := time.NewTimer(REPORT_TIMEOUT) + defer tmr.Stop() + go func() { + <-tmr.C + logger.WithFields(logrus.Fields{ + "prefix": prefix, + "limit": limit, + }).Warnf("%s call took longer than %v", utils.GetCurrentFuncName(), REPORT_TIMEOUT) + }() + ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(time.Second*30)) defer cancel() @@ -2874,6 +3105,17 @@ func (bigtable *Bigtable) GetEth1ERC1155ForAddress(prefix string, limit int64) ( } func (bigtable *Bigtable) GetAddressErc1155TableData(address string, search string, pageToken string) (*types.DataTableResponse, error) { + tmr := time.NewTimer(REPORT_TIMEOUT) + defer tmr.Stop() + go func() { + <-tmr.C + logger.WithFields(logrus.Fields{ + "address": address, + "search": search, + "pageToken": pageToken, + }).Warnf("%s call took longer than %v", utils.GetCurrentFuncName(), REPORT_TIMEOUT) + }() + if pageToken == "" { pageToken = fmt.Sprintf("%s:I:ERC1155:%s:%s:", bigtable.chainId, address, FILTER_TIME) } @@ -2921,6 +3163,16 @@ func (bigtable *Bigtable) GetAddressErc1155TableData(address string, search stri } func (bigtable *Bigtable) GetMetadataUpdates(prefix string, startToken string, limit int) ([]string, []*types.Eth1AddressBalance, error) { + tmr := time.NewTimer(REPORT_TIMEOUT) + defer tmr.Stop() + go func() { + <-tmr.C + logger.WithFields(logrus.Fields{ + "prefix": prefix, + "startToken": startToken, + }).Warnf("%s call took longer than %v", utils.GetCurrentFuncName(), REPORT_TIMEOUT) + }() + ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(time.Minute*120)) defer cancel() @@ -2948,6 +3200,16 @@ func (bigtable *Bigtable) GetMetadataUpdates(prefix string, startToken string, l } func (bigtable *Bigtable) GetMetadata(startToken string, limit int) ([]string, []*types.Eth1AddressBalance, error) { + tmr := time.NewTimer(REPORT_TIMEOUT) + defer tmr.Stop() + go func() { + <-tmr.C + logger.WithFields(logrus.Fields{ + "startToken": startToken, + "limit": limit, + }).Warnf("%s call took longer than %v", utils.GetCurrentFuncName(), REPORT_TIMEOUT) + }() + ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(time.Minute*120)) defer cancel() @@ -2977,6 +3239,15 @@ func (bigtable *Bigtable) GetMetadata(startToken string, limit int) ([]string, [ } func (bigtable *Bigtable) GetMetadataForAddress(address []byte) (*types.Eth1AddressMetadata, error) { + tmr := time.NewTimer(REPORT_TIMEOUT) + defer tmr.Stop() + go func() { + <-tmr.C + logger.WithFields(logrus.Fields{ + "address": address, + }).Warnf("%s call took longer than %v", utils.GetCurrentFuncName(), REPORT_TIMEOUT) + }() + ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(time.Second*30)) defer cancel() @@ -3078,6 +3349,16 @@ func (bigtable *Bigtable) GetMetadataForAddress(address []byte) (*types.Eth1Addr } func (bigtable *Bigtable) GetBalanceForAddress(address []byte, token []byte) (*types.Eth1AddressBalance, error) { + tmr := time.NewTimer(REPORT_TIMEOUT) + defer tmr.Stop() + go func() { + <-tmr.C + logger.WithFields(logrus.Fields{ + "address": address, + "token": token, + }).Warnf("%s call took longer than %v", utils.GetCurrentFuncName(), REPORT_TIMEOUT) + }() + ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(time.Second*30)) defer cancel() @@ -3115,6 +3396,14 @@ func (bigtable *Bigtable) GetBalanceForAddress(address []byte, token []byte) (*t } func (bigtable *Bigtable) GetERC20MetadataForAddress(address []byte) (*types.ERC20Metadata, error) { + tmr := time.NewTimer(REPORT_TIMEOUT) + defer tmr.Stop() + go func() { + <-tmr.C + logger.WithFields(logrus.Fields{ + "address": address, + }).Warnf("%s call took longer than %v", utils.GetCurrentFuncName(), REPORT_TIMEOUT) + }() if len(address) == 1 { return &types.ERC20Metadata{ @@ -3243,6 +3532,15 @@ func (bigtable *Bigtable) SaveERC20Metadata(address []byte, metadata *types.ERC2 } func (bigtable *Bigtable) GetAddressName(address []byte) (string, error) { + tmr := time.NewTimer(REPORT_TIMEOUT) + defer tmr.Stop() + go func() { + <-tmr.C + logger.WithFields(logrus.Fields{ + "address": address, + }).Warnf("%s call took longer than %v", utils.GetCurrentFuncName(), REPORT_TIMEOUT) + }() + ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(time.Second*30)) defer cancel() @@ -3276,6 +3574,15 @@ func (bigtable *Bigtable) GetAddressName(address []byte) (string, error) { } func (bigtable *Bigtable) GetAddressNames(addresses map[string]string) error { + tmr := time.NewTimer(REPORT_TIMEOUT) + defer tmr.Stop() + go func() { + <-tmr.C + logger.WithFields(logrus.Fields{ + "addresses": addresses, + }).Warnf("%s call took longer than %v", utils.GetCurrentFuncName(), REPORT_TIMEOUT) + }() + if len(addresses) == 0 { return nil } @@ -3319,6 +3626,15 @@ func (bigtable *Bigtable) SaveAddressName(address []byte, name string) error { } func (bigtable *Bigtable) GetContractMetadata(address []byte) (*types.ContractMetadata, error) { + tmr := time.NewTimer(REPORT_TIMEOUT) + defer tmr.Stop() + go func() { + <-tmr.C + logger.WithFields(logrus.Fields{ + "address": address, + }).Warnf("%s call took longer than %v", utils.GetCurrentFuncName(), REPORT_TIMEOUT) + }() + ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(time.Second*30)) defer cancel() @@ -3498,6 +3814,16 @@ func (bigtable *Bigtable) SaveBlockKeys(blockNumber uint64, blockHash []byte, ke } func (bigtable *Bigtable) GetBlockKeys(blockNumber uint64, blockHash []byte) ([]string, error) { + tmr := time.NewTimer(REPORT_TIMEOUT) + defer tmr.Stop() + go func() { + <-tmr.C + logger.WithFields(logrus.Fields{ + "blockNumber": blockNumber, + "blockHash": blockHash, + }).Warnf("%s call took longer than %v", utils.GetCurrentFuncName(), REPORT_TIMEOUT) + }() + ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(time.Second*30)) defer cancel() @@ -3559,6 +3885,16 @@ func (bigtable *Bigtable) DeleteBlock(blockNumber uint64, blockHash []byte) erro } func (bigtable *Bigtable) GetEth1TxForToken(prefix string, limit int64) ([]*types.Eth1ERC20Indexed, string, error) { + tmr := time.NewTimer(REPORT_TIMEOUT) + defer tmr.Stop() + go func() { + <-tmr.C + logger.WithFields(logrus.Fields{ + "prefix": prefix, + "limit": limit, + }).Warnf("%s call took longer than %v", utils.GetCurrentFuncName(), REPORT_TIMEOUT) + }() + ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(time.Second*30)) defer cancel() @@ -3669,6 +4005,16 @@ func (bigtable *Bigtable) GetTokenTransactionsTableData(token []byte, address [] } func (bigtable *Bigtable) SearchForAddress(addressPrefix []byte, limit int) ([]*types.Eth1AddressSearchItem, error) { + tmr := time.NewTimer(REPORT_TIMEOUT) + defer tmr.Stop() + go func() { + <-tmr.C + logger.WithFields(logrus.Fields{ + "addressPrefix": addressPrefix, + "limit": limit, + }).Warnf("%s call took longer than %v", utils.GetCurrentFuncName(), REPORT_TIMEOUT) + }() + ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(time.Second*30)) defer cancel() @@ -3713,6 +4059,15 @@ func getSignaturePrefix(st types.SignatureType) string { // Get the status of the last signature import run func (bigtable *Bigtable) GetSignatureImportStatus(st types.SignatureType) (*types.SignatureImportStatus, error) { + tmr := time.NewTimer(REPORT_TIMEOUT) + defer tmr.Stop() + go func() { + <-tmr.C + logger.WithFields(logrus.Fields{ + "st": st, + }).Warnf("%s call took longer than %v", utils.GetCurrentFuncName(), REPORT_TIMEOUT) + }() + ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(time.Second*30)) defer cancel() key := fmt.Sprintf("1:%v_SIGNATURE_IMPORT_STATUS", getSignaturePrefix(st)) @@ -3794,6 +4149,16 @@ func (bigtable *Bigtable) SaveSignatures(signatures []types.Signature, st types. // get a signature by it's hex representation func (bigtable *Bigtable) GetSignature(hex string, st types.SignatureType) (*string, error) { + tmr := time.NewTimer(REPORT_TIMEOUT) + defer tmr.Stop() + go func() { + <-tmr.C + logger.WithFields(logrus.Fields{ + "hex": hex, + "st": st, + }).Warnf("%s call took longer than %v", utils.GetCurrentFuncName(), REPORT_TIMEOUT) + }() + ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(time.Second*30)) defer cancel() key := fmt.Sprintf("1:%v_SIGNATURE:%v", getSignaturePrefix(st), hex) @@ -3915,6 +4280,16 @@ func (bigtable *Bigtable) SaveGasNowHistory(slow, standard, rapid, fast *big.Int } func (bigtable *Bigtable) GetGasNowHistory(ts, pastTs time.Time) ([]types.GasNowHistory, error) { + tmr := time.NewTimer(REPORT_TIMEOUT) + defer tmr.Stop() + go func() { + <-tmr.C + logger.WithFields(logrus.Fields{ + "ts": ts, + "pastTs": pastTs, + }).Warnf("%s call took longer than %v", utils.GetCurrentFuncName(), REPORT_TIMEOUT) + }() + ctx, done := context.WithTimeout(context.Background(), time.Second*30) defer done() diff --git a/db/bigtable_init.go b/db/bigtable_init.go new file mode 100644 index 0000000000..022aecef30 --- /dev/null +++ b/db/bigtable_init.go @@ -0,0 +1,90 @@ +package db + +import ( + "context" + "eth2-exporter/utils" + "fmt" + "time" + + gcp_bigtable "cloud.google.com/go/bigtable" +) + +func InitBigtableSchema() error { + + tables := make(map[string]map[string]gcp_bigtable.GCPolicy) + + tables["beaconchain_validators"] = map[string]gcp_bigtable.GCPolicy{ + ATTESTATIONS_FAMILY: gcp_bigtable.MaxVersionsGCPolicy(1), + } + tables["beaconchain_validators_history"] = map[string]gcp_bigtable.GCPolicy{ + VALIDATOR_BALANCES_FAMILY: nil, + VALIDATOR_HIGHEST_ACTIVE_INDEX_FAMILY: nil, + ATTESTATIONS_FAMILY: nil, + PROPOSALS_FAMILY: nil, + SYNC_COMMITTEES_FAMILY: nil, + SYNC_COMMITTEES_PARTICIPATION_FAMILY: nil, + INCOME_DETAILS_COLUMN_FAMILY: nil, + STATS_COLUMN_FAMILY: nil, + } + tables["blocks"] = map[string]gcp_bigtable.GCPolicy{ + DEFAULT_FAMILY_BLOCKS: gcp_bigtable.MaxVersionsGCPolicy(1), + } + tables["data"] = map[string]gcp_bigtable.GCPolicy{ + CONTRACT_METADATA_FAMILY: gcp_bigtable.MaxAgeGCPolicy(time.Hour * 24), + DEFAULT_FAMILY: nil, + } + tables["machine_metrics"] = map[string]gcp_bigtable.GCPolicy{ + MACHINE_METRICS_COLUMN_FAMILY: gcp_bigtable.MaxAgeGCPolicy(time.Hour * 24 * 31), + } + tables["metadata"] = map[string]gcp_bigtable.GCPolicy{ + ACCOUNT_METADATA_FAMILY: nil, + CONTRACT_METADATA_FAMILY: nil, + ERC20_METADATA_FAMILY: nil, + ERC721_METADATA_FAMILY: nil, + ERC1155_METADATA_FAMILY: nil, + SERIES_FAMILY: gcp_bigtable.MaxVersionsGCPolicy(1), + } + tables["metadata_updates"] = map[string]gcp_bigtable.GCPolicy{ + METADATA_UPDATES_FAMILY_BLOCKS: gcp_bigtable.MaxAgeGCPolicy(time.Hour * 24), + DEFAULT_FAMILY: nil, + } + ctx, cancel := context.WithTimeout(context.Background(), time.Second*30) + defer cancel() + + admin, err := gcp_bigtable.NewAdminClient(ctx, utils.Config.Bigtable.Project, utils.Config.Bigtable.Instance) + if err != nil { + return err + } + + existingTables, err := admin.Tables(ctx) + if err != nil { + return err + } + + if len(existingTables) > 0 { + return fmt.Errorf("aborting bigtable schema init as tables are already present") + } + + for name, definition := range tables { + err := admin.CreateTable(ctx, name) + if err != nil { + return err + } + + for columnFamily, gcPolicy := range definition { + err := admin.CreateColumnFamily(ctx, name, columnFamily) + if err != nil { + return err + } + + if gcPolicy != nil { + err := admin.SetGCPolicy(ctx, name, columnFamily, gcPolicy) + if err != nil { + return err + } + } + } + } + + return nil +} diff --git a/db/db.go b/db/db.go index 7694a2f1aa..35e7ebff2f 100644 --- a/db/db.go +++ b/db/db.go @@ -5,6 +5,8 @@ import ( "crypto/sha1" "database/sql" "embed" + "encoding/hex" + "errors" "eth2-exporter/metrics" "eth2-exporter/types" "eth2-exporter/utils" @@ -48,10 +50,14 @@ var saveValidatorsMux = &sync.Mutex{} var farFutureEpoch = uint64(18446744073709551615) var maxSqlNumber = uint64(9223372036854775807) +const WithdrawalsQueryLimit = 10000 +const BlsChangeQueryLimit = 10000 const MaxSqlInteger = 2147483647 -var addressLikeRE = regexp.MustCompile(`^(0x)?[0-9a-fA-F]{0,40}$`) -var blsLikeRE = regexp.MustCompile(`^(0x)?[0-9a-fA-F]{0,96}$`) +var addressRE = regexp.MustCompile(`^(0x)?[0-9a-fA-F]{40}$`) +var blsRE = regexp.MustCompile(`^(0x)?[0-9a-fA-F]{96}$`) + +var ErrNoStats = errors.New("no stats available") func dbTestConnection(dbConn *sqlx.DB, dataBaseName string) { // The golang sql driver does not properly implement PingContext @@ -512,16 +518,19 @@ func GetAllEpochs() ([]uint64, error) { return epochs, nil } -// Count finalized epochs in range (including start and end epoch) -func CountFinalizedEpochs(startEpoch uint64, endEpoch uint64) (uint64, error) { - var count uint64 - err := WriterDb.Get(&count, "SELECT COUNT(*) FROM epochs WHERE epoch >= $1 AND epoch <= $2 AND finalized", startEpoch, endEpoch) - +// Get latest finalized epoch +func GetLatestFinalizedEpoch() (uint64, error) { + var latestFinalized uint64 + err := WriterDb.Get(&latestFinalized, "SELECT epoch FROM epochs WHERE finalized ORDER BY epoch DESC LIMIT 1") if err != nil { - return 0, fmt.Errorf("error counting finalized epochs [%v -> %v] from DB: %w", startEpoch, endEpoch, err) + if err == sql.ErrNoRows { + return 0, nil + } + utils.LogError(err, "error retrieving latest exported finalized epoch from the database", 0) + return 0, err } - return count, nil + return latestFinalized, nil } // GetLastPendingAndProposedBlocks will return all proposed and pending blocks (ignores missed slots) from the database @@ -706,7 +715,7 @@ func SaveValidatorQueue(validators *types.ValidatorQueue) error { return err } -func SaveBlock(block *types.Block) error { +func SaveBlock(block *types.Block, forceSlotUpdate bool) error { blocksMap := make(map[uint64]map[string]*types.Block) if blocksMap[block.Slot] == nil { @@ -721,7 +730,7 @@ func SaveBlock(block *types.Block) error { defer tx.Rollback() logger.Infof("exporting block data") - err = saveBlocks(blocksMap, tx) + err = saveBlocks(blocksMap, tx, forceSlotUpdate) if err != nil { logger.Fatalf("error saving blocks to db: %v", err) return fmt.Errorf("error saving blocks to db: %w", err) @@ -819,13 +828,13 @@ func SaveEpoch(data *types.EpochData, client rpc.Client) error { logger.WithFields(logrus.Fields{"chainEpoch": utils.TimeToEpoch(time.Now()), "exportEpoch": data.Epoch}).Infof("starting export of epoch %v", data.Epoch) logger.Infof("exporting block data") - err = saveBlocks(data.Blocks, tx) + err = saveBlocks(data.Blocks, tx, false) if err != nil { logger.Fatalf("error saving blocks to db: %v", err) return fmt.Errorf("error saving blocks to db: %w", err) } - if uint64(utils.TimeToEpoch(time.Now())) > data.Epoch+10 { + if data.Epoch%10 != 0 && uint64(utils.TimeToEpoch(time.Now())) > data.Epoch+10 { logger.WithFields(logrus.Fields{"exportEpoch": data.Epoch, "chainEpoch": utils.TimeToEpoch(time.Now())}).Infof("skipping exporting validators because epoch is far behind head") } else { go func() { @@ -841,7 +850,7 @@ func SaveEpoch(data *types.EpochData, client rpc.Client) error { } defer validatorsTx.Rollback() - err = saveValidators(data, validatorsTx, client) + err = SaveValidators(data, validatorsTx, client, 10000) if err != nil { logger.Errorf("error saving validators to db: %v", err) } @@ -895,11 +904,6 @@ func SaveEpoch(data *types.EpochData, client rpc.Client) error { validatorBalanceAverage := new(big.Int).Div(validatorBalanceSum, new(big.Int).SetInt64(int64(validatorsCount))) - finalized := false - if data.Epoch == 0 { - finalized = true - } - _, err = tx.Exec(` INSERT INTO epochs ( epoch, @@ -913,10 +917,10 @@ func SaveEpoch(data *types.EpochData, client rpc.Client) error { validatorscount, averagevalidatorbalance, totalvalidatorbalance, - finalized, eligibleether, globalparticipationrate, - votedether + votedether, + finalized ) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15) ON CONFLICT (epoch) DO UPDATE SET @@ -932,7 +936,8 @@ func SaveEpoch(data *types.EpochData, client rpc.Client) error { totalvalidatorbalance = excluded.totalvalidatorbalance, eligibleether = excluded.eligibleether, globalparticipationrate = excluded.globalparticipationrate, - votedether = excluded.votedether`, + votedether = excluded.votedether, + finalized = excluded.finalized`, data.Epoch, len(data.Blocks), proposerSlashingsCount, @@ -944,10 +949,10 @@ func SaveEpoch(data *types.EpochData, client rpc.Client) error { validatorsCount, validatorBalanceAverage.Uint64(), validatorBalanceSum.Uint64(), - finalized, data.EpochParticipationStats.EligibleEther, data.EpochParticipationStats.GlobalParticipationRate, - data.EpochParticipationStats.VotedEther) + data.EpochParticipationStats.VotedEther, + data.Finalized) if err != nil { return fmt.Errorf("error executing save epoch statement: %w", err) @@ -1045,17 +1050,27 @@ func saveGraffitiwall(blocks map[uint64]map[string]*types.Block, tx *sqlx.Tx) er return nil } -func saveValidators(data *types.EpochData, tx *sqlx.Tx, client rpc.Client) error { +func SaveValidators(data *types.EpochData, tx *sqlx.Tx, client rpc.Client, activationBalanceBatchSize int) error { start := time.Now() defer func() { metrics.TaskDuration.WithLabelValues("db_save_validators").Observe(time.Since(start).Seconds()) }() + if activationBalanceBatchSize <= 0 { + activationBalanceBatchSize = 10000 + } + var genesisBalances map[uint64][]*types.ValidatorBalance if data.Epoch == 0 { var err error - genesisBalances, err = BigtableClient.GetValidatorBalanceHistory([]uint64{}, 0, 0) + + indices := make([]uint64, 0, len(data.Validators)) + + for _, validator := range data.Validators { + indices = append(indices, validator.Index) + } + genesisBalances, err = BigtableClient.GetValidatorBalanceHistory(indices, 0, 0) if err != nil { return err } @@ -1099,7 +1114,7 @@ func saveValidators(data *types.EpochData, tx *sqlx.Tx, client rpc.Client) error err := tx.Select(¤tState, "SELECT validatorindex, withdrawableepoch, withdrawalcredentials, slashed, activationeligibilityepoch, activationepoch, exitepoch, status FROM validators;") if err != nil { - return err + return fmt.Errorf("error retrieving current validator state set: %v", err) } lastAttestationSlots, err := BigtableClient.GetLastAttestationSlots([]uint64{}) @@ -1126,6 +1141,25 @@ func saveValidators(data *types.EpochData, tx *sqlx.Tx, client rpc.Client) error var queries strings.Builder + insertStmt, err := tx.Prepare(`INSERT INTO validators ( + validatorindex, + pubkey, + withdrawableepoch, + withdrawalcredentials, + balance, + effectivebalance, + slashed, + activationeligibilityepoch, + activationepoch, + exitepoch, + pubkeyhex, + status + ) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12);`) + if err != nil { + return fmt.Errorf("error preparing insert validator statement: %w", err) + } + updates := 0 for _, v := range data.Validators { @@ -1146,23 +1180,11 @@ func saveValidators(data *types.EpochData, tx *sqlx.Tx, client rpc.Client) error c := currentStateMap[v.Index] if c == nil { - logger.Infof("validator %v is new", v.Index) - - _, err = tx.Exec(`INSERT INTO validators ( - validatorindex, - pubkey, - withdrawableepoch, - withdrawalcredentials, - balance, - effectivebalance, - slashed, - activationeligibilityepoch, - activationepoch, - exitepoch, - pubkeyhex, - status - ) - VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12);`, + if v.Index%1000 == 0 { + logger.Infof("validator %v is new", v.Index) + } + + _, err = insertStmt.Exec( v.Index, v.PublicKey, v.WithdrawableEpoch, @@ -1267,6 +1289,11 @@ func saveValidators(data *types.EpochData, tx *sqlx.Tx, client rpc.Client) error } } + err = insertStmt.Close() + if err != nil { + return fmt.Errorf("error closing insert validator statement: %w", err) + } + if updates > 0 { updateStart := time.Now() logger.Infof("applying %v update queries", updates) @@ -1284,7 +1311,7 @@ func saveValidators(data *types.EpochData, tx *sqlx.Tx, client rpc.Client) error ActivationEpoch uint64 }{} - err = tx.Select(&newValidators, "SELECT validatorindex, activationepoch FROM validators WHERE balanceactivation IS NULL ORDER BY activationepoch LIMIT 10000") + err = tx.Select(&newValidators, "SELECT validatorindex, activationepoch FROM validators WHERE balanceactivation IS NULL ORDER BY activationepoch LIMIT $1", activationBalanceBatchSize) if err != nil { return err } @@ -1399,7 +1426,7 @@ func GetRelayDataForIndexedBlocks(blocks []*types.Eth1BlockIndexed) (map[common. return relaysDataMap, nil } -func saveBlocks(blocks map[uint64]map[string]*types.Block, tx *sqlx.Tx) error { +func saveBlocks(blocks map[uint64]map[string]*types.Block, tx *sqlx.Tx, forceSlotUpdate bool) error { start := time.Now() defer func() { metrics.TaskDuration.WithLabelValues("db_save_blocks").Observe(time.Since(start).Seconds()) @@ -1420,8 +1447,8 @@ func saveBlocks(blocks map[uint64]map[string]*types.Block, tx *sqlx.Tx) error { defer stmtBlock.Close() stmtWithdrawals, err := tx.Prepare(` - INSERT INTO blocks_withdrawals (block_slot, block_root, withdrawalindex, validatorindex, address, address_text, amount) - VALUES ($1, $2, $3, $4, $5, $6, $7) + INSERT INTO blocks_withdrawals (block_slot, block_root, withdrawalindex, validatorindex, address, amount) + VALUES ($1, $2, $3, $4, $5, $6) ON CONFLICT (block_slot, block_root, withdrawalindex) DO NOTHING`) if err != nil { return err @@ -1429,8 +1456,8 @@ func saveBlocks(blocks map[uint64]map[string]*types.Block, tx *sqlx.Tx) error { defer stmtWithdrawals.Close() stmtBLSChange, err := tx.Prepare(` - INSERT INTO blocks_bls_change (block_slot, block_root, validatorindex, signature, pubkey, pubkey_text, address) - VALUES ($1, $2, $3, $4, $5, $6, $7) + INSERT INTO blocks_bls_change (block_slot, block_root, validatorindex, signature, pubkey, address) + VALUES ($1, $2, $3, $4, $5, $6) ON CONFLICT (block_slot, block_root, validatorindex) DO NOTHING`) if err != nil { return err @@ -1513,13 +1540,15 @@ func saveBlocks(blocks map[uint64]map[string]*types.Block, tx *sqlx.Tx) error { start := time.Now() blockLog := logger.WithFields(logrus.Fields{"slot": b.Slot, "blockRoot": fmt.Sprintf("%x", b.BlockRoot)}) - var dbBlockRootHash []byte - err := WriterDb.Get(&dbBlockRootHash, "SELECT blockroot FROM blocks WHERE slot = $1 and blockroot = $2", b.Slot, b.BlockRoot) - if err == nil && bytes.Equal(dbBlockRootHash, b.BlockRoot) { - blockLog.Infof("skipping export of block as it is already present in the db") - continue - } else if err != nil && err != sql.ErrNoRows { - return fmt.Errorf("error checking for block in db: %w", err) + if !forceSlotUpdate { + var dbBlockRootHash []byte + err := WriterDb.Get(&dbBlockRootHash, "SELECT blockroot FROM blocks WHERE slot = $1 and blockroot = $2", b.Slot, b.BlockRoot) + if err == nil && bytes.Equal(dbBlockRootHash, b.BlockRoot) { + blockLog.Infof("skipping export of block as it is already present in the db") + continue + } else if err != nil && err != sql.ErrNoRows { + return fmt.Errorf("error checking for block in db: %w", err) + } } blockLog.WithField("duration", time.Since(start)).Tracef("check if exists") @@ -1597,7 +1626,7 @@ func saveBlocks(blocks map[uint64]map[string]*types.Block, tx *sqlx.Tx) error { b.Signature, b.RandaoReveal, b.Graffiti, - utils.GraffitiToSring(b.Graffiti), + utils.GraffitiToString(b.Graffiti), b.Eth1Data.DepositRoot, b.Eth1Data.DepositCount, b.Eth1Data.BlockHash, @@ -1647,7 +1676,7 @@ func saveBlocks(blocks map[uint64]map[string]*types.Block, tx *sqlx.Tx) error { logger.Tracef("writing transactions and withdrawal data") if payload := b.ExecutionPayload; payload != nil { for _, w := range payload.Withdrawals { - _, err := stmtWithdrawals.Exec(b.Slot, b.BlockRoot, w.Index, w.ValidatorIndex, w.Address, fmt.Sprintf("%x", w.Address), w.Amount) + _, err := stmtWithdrawals.Exec(b.Slot, b.BlockRoot, w.Index, w.ValidatorIndex, w.Address, w.Amount) if err != nil { return fmt.Errorf("error executing stmtWithdrawals for block at slot %v: %w", b.Slot, err) } @@ -1667,7 +1696,7 @@ func saveBlocks(blocks map[uint64]map[string]*types.Block, tx *sqlx.Tx) error { n = time.Now() logger.Tracef("writing bls change data") for _, bls := range b.SignedBLSToExecutionChange { - _, err := stmtBLSChange.Exec(b.Slot, b.BlockRoot, bls.Message.Validatorindex, bls.Signature, bls.Message.BlsPubkey, fmt.Sprintf("%x", bls.Message.BlsPubkey), bls.Message.Address) + _, err := stmtBLSChange.Exec(b.Slot, b.BlockRoot, bls.Message.Validatorindex, bls.Signature, bls.Message.BlsPubkey, bls.Message.Address) if err != nil { return fmt.Errorf("error executing stmtBLSChange for block %v: %w", b.Slot, err) } @@ -1744,28 +1773,19 @@ func UpdateEpochStatus(stats *types.ValidatorParticipation) error { UPDATE epochs SET eligibleether = $1, globalparticipationrate = $2, - votedether = $3 - WHERE epoch = $4`, - stats.EligibleEther, stats.GlobalParticipationRate, stats.VotedEther, stats.Epoch) + votedether = $3, + finalized = $4 + WHERE epoch = $5`, + stats.EligibleEther, stats.GlobalParticipationRate, stats.VotedEther, stats.Finalized, stats.Epoch) return err } -// UpdateEpochFinalization will update finalized-flag of unfinalized epochs -func UpdateEpochFinalization(finality_epoch uint64) error { - // to prevent a full table scan, the query is constrained to update only between the last epoch that was tagged finalized and the passed finality_epoch - // will not fill gaps in the db in finalization this way, but makes the query much faster. - _, err := WriterDb.Exec(` - UPDATE epochs - SET finalized = true - WHERE epoch BETWEEN COALESCE(( - SELECT epoch - FROM epochs - WHERE finalized = true - ORDER BY epoch DESC - LIMIT 1 - ),0) AND $1`, finality_epoch) - return err +// GetValidatorIndices will return the total-validator-indices +func GetValidatorIndices() ([]uint64, error) { + indices := []uint64{} + err := ReaderDb.Select(&indices, "select validatorindex from validators order by validatorindex;") + return indices, err } // GetTotalValidatorsCount will return the total-validator-count @@ -2066,7 +2086,11 @@ func GetSlotVizData(latestEpoch uint64) ([]*types.SlotVizEpochs, error) { latestEpoch = 0 } - err := ReaderDb.Select(&blks, ` + latestFinalizedEpoch, err := GetLatestFinalizedEpoch() + if err != nil { + return nil, err + } + err = ReaderDb.Select(&blks, ` SELECT b.slot, b.blockroot, @@ -2079,12 +2103,12 @@ func GetSlotVizData(latestEpoch uint64) ([]*types.SlotVizEpochs, error) { END AS status, b.epoch, COALESCE(e.globalparticipationrate, 0) AS globalparticipationrate, - COALESCE(e.finalized, false) AS finalized + (b.epoch <= $2) AS finalized FROM blocks b LEFT JOIN epochs e ON e.epoch = b.epoch WHERE b.epoch >= $1 ORDER BY slot DESC; -`, latestEpoch) +`, latestEpoch, latestFinalizedEpoch) if err != nil { return nil, err } @@ -2235,30 +2259,40 @@ func GetTotalWithdrawals() (total uint64, err error) { } func GetWithdrawalsCountForQuery(query string) (uint64, error) { + t0 := time.Now() + defer func() { + logger.WithFields(logrus.Fields{"duration": time.Since(t0)}).Infof("finished GetWithdrawalsCountForQuery") + }() count := uint64(0) withdrawalsQuery := ` - SELECT count(*) - FROM blocks_withdrawals w - INNER JOIN blocks b ON w.block_root = b.blockroot AND b.status = '1' - %s` + SELECT COUNT(*) FROM ( + SELECT b.slot + FROM blocks_withdrawals w + INNER JOIN blocks b ON w.block_root = b.blockroot AND b.status = '1' + %s + LIMIT %d + ) a` var err error = nil trimmedQuery := strings.ToLower(strings.TrimPrefix(query, "0x")) - // Check whether the query can be used for a validator, slot or epoch search - if uiQuery, parseErr := strconv.ParseUint(query, 10, 64); parseErr == nil { + if addressRE.MatchString(query) { + searchQuery := `WHERE w.address = $1` + addr, decErr := hex.DecodeString(trimmedQuery) + if err != nil { + return 0, decErr + } + err = ReaderDb.Get(&count, fmt.Sprintf(withdrawalsQuery, searchQuery, WithdrawalsQueryLimit), + addr) + } else if uiQuery, parseErr := strconv.ParseUint(query, 10, 64); parseErr == nil { + // Check whether the query can be used for a validator, slot or epoch search searchQuery := ` - WHERE w.validatorindex = $1 - OR block_slot = $1 - OR (block_slot / $3) = $1 - OR address_text LIKE ($2 || '%')` - err = ReaderDb.Get(&count, fmt.Sprintf(withdrawalsQuery, searchQuery), - uiQuery, trimmedQuery, utils.Config.Chain.ClConfig.SlotsPerEpoch) - } else if addressLikeRE.MatchString(query) { - searchQuery := `WHERE address_text LIKE ($1 || '%')` - err = ReaderDb.Get(&count, fmt.Sprintf(withdrawalsQuery, searchQuery), - trimmedQuery) + WHERE w.validatorindex = $1 + OR w.block_slot = $1 + OR w.block_slot BETWEEN $1*$2 AND ($1+1)*$2-1` + err = ReaderDb.Get(&count, fmt.Sprintf(withdrawalsQuery, searchQuery, WithdrawalsQueryLimit), + uiQuery, utils.Config.Chain.ClConfig.SlotsPerEpoch) } if err != nil { @@ -2269,6 +2303,10 @@ func GetWithdrawalsCountForQuery(query string) (uint64, error) { } func GetWithdrawals(query string, length, start uint64, orderBy, orderDir string) ([]*types.Withdrawals, error) { + t0 := time.Now() + defer func() { + logger.WithFields(logrus.Fields{"duration": time.Since(t0)}).Infof("finished GetWithdrawals") + }() withdrawals := []*types.Withdrawals{} if orderDir != "desc" && orderDir != "asc" { @@ -2295,7 +2333,7 @@ func GetWithdrawals(query string, length, start uint64, orderBy, orderDir string w.amount FROM blocks_withdrawals w INNER JOIN blocks b ON w.block_root = b.blockroot AND b.status = '1' - %s + %s ORDER BY %s %s LIMIT $1 OFFSET $2` @@ -2304,21 +2342,22 @@ func GetWithdrawals(query string, length, start uint64, orderBy, orderDir string trimmedQuery := strings.ToLower(strings.TrimPrefix(query, "0x")) if trimmedQuery != "" { - // Check whether the query can be used for a validator, slot or epoch search - if uiQuery, parseErr := strconv.ParseUint(query, 10, 64); parseErr == nil { + if addressRE.MatchString(query) { + searchQuery := `WHERE w.address = $3` + addr, decErr := hex.DecodeString(trimmedQuery) + if decErr != nil { + return nil, decErr + } + err = ReaderDb.Select(&withdrawals, fmt.Sprintf(withdrawalsQuery, searchQuery, orderBy, orderDir), + length, start, addr) + } else if uiQuery, parseErr := strconv.ParseUint(query, 10, 64); parseErr == nil { + // Check whether the query can be used for a validator, slot or epoch search searchQuery := ` WHERE w.validatorindex = $3 - OR block_slot = $3 - OR (block_slot / $5) = $3 - OR address_text LIKE ($4 || '%')` - - err = ReaderDb.Select(&withdrawals, fmt.Sprintf(withdrawalsQuery, searchQuery, orderBy, orderDir), - length, start, uiQuery, trimmedQuery, utils.Config.Chain.ClConfig.SlotsPerEpoch) - } else if addressLikeRE.MatchString(query) { - searchQuery := `WHERE address_text LIKE ($3 || '%')` - + OR w.block_slot = $3 + OR w.block_slot BETWEEN $3*$4 AND ($3+1)*$4-1` err = ReaderDb.Select(&withdrawals, fmt.Sprintf(withdrawalsQuery, searchQuery, orderBy, orderDir), - length, start, trimmedQuery) + length, start, uiQuery, utils.Config.Chain.ClConfig.SlotsPerEpoch) } } else { err = ReaderDb.Select(&withdrawals, fmt.Sprintf(withdrawalsQuery, "", orderBy, orderDir), length, start) @@ -2540,6 +2579,10 @@ func GetAddressWithdrawalsTotal(address []byte) (uint64, error) { var total uint64 err := ReaderDb.Get(&total, ` + /*+ + BitmapScan(w) + NestLoop(b w) + */ SELECT COALESCE(sum(w.amount), 0) as total FROM blocks_withdrawals w @@ -2559,6 +2602,10 @@ func GetDashboardWithdrawalsCount(validators []uint64) (uint64, error) { var count uint64 validatorFilter := pq.Array(validators) err := ReaderDb.Get(&count, ` + /*+ + BitmapScan(w) + NestLoop(b w) + */ SELECT count(*) FROM blocks_withdrawals w INNER JOIN blocks b ON b.blockroot = w.block_root AND b.status = '1' @@ -2580,6 +2627,10 @@ func GetDashboardWithdrawals(validators []uint64, limit uint64, offset uint64, o } validatorFilter := pq.Array(validators) err := ReaderDb.Select(&withdrawals, fmt.Sprintf(` + /*+ + BitmapScan(w) + NestLoop(b w) + */ SELECT w.block_slot as slot, w.withdrawalindex as index, @@ -2610,6 +2661,10 @@ func GetValidatorWithdrawalsCount(validator uint64) (count, lastWithdrawalEpoch r := &dbResponse{} err = ReaderDb.Get(r, ` + /*+ + BitmapScan(w) + NestLoop(b w) + */ SELECT count(*) as withdrawals_count, COALESCE(max(block_slot), 0) as last_withdawal_slot FROM blocks_withdrawals w INNER JOIN blocks b ON b.blockroot = w.block_root AND b.status = '1' @@ -2889,41 +2944,34 @@ func GetBLSChangesCountForQuery(query string) (uint64, error) { count := uint64(0) blsQuery := ` - SELECT COUNT(*) - FROM blocks_bls_change bls - INNER JOIN blocks b ON bls.block_root = b.blockroot AND b.status = '1' - %s - %s` + SELECT COUNT(*) FROM ( + SELECT b.slot + FROM blocks_bls_change bls + INNER JOIN blocks b ON bls.block_root = b.blockroot AND b.status = '1' + %s + LIMIT %d + ) a + ` trimmedQuery := strings.ToLower(strings.TrimPrefix(query, "0x")) var err error = nil - joinQuery := ` - LEFT JOIN ( - SELECT - validators.validatorindex as validatorindex, - eth1_deposits.from_address_text as deposit_address_text - FROM validators - INNER JOIN eth1_deposits ON validators.pubkey = eth1_deposits.publickey - ) AS val ON val.validatorindex = bls.validatorindex` - - // Check whether the query can be used for a validator, slot or epoch search - if uiQuery, parseErr := strconv.ParseUint(query, 10, 64); parseErr == nil { + if blsRE.MatchString(query) { + searchQuery := `WHERE bls.pubkey = $1` + pubkey, decErr := hex.DecodeString(trimmedQuery) + if decErr != nil { + return 0, decErr + } + err = ReaderDb.Get(&count, fmt.Sprintf(blsQuery, searchQuery, BlsChangeQueryLimit), + pubkey) + } else if uiQuery, parseErr := strconv.ParseUint(query, 10, 64); parseErr == nil { + // Check whether the query can be used for a validator, slot or epoch search searchQuery := ` WHERE bls.validatorindex = $1 - OR block_slot = $1 - OR (block_slot / $3) = $1 - OR pubkey_text LIKE ($2 || '%') - OR deposit_address_text LIKE ($2 || '%')` - - err = ReaderDb.Get(&count, fmt.Sprintf(blsQuery, joinQuery, searchQuery), - uiQuery, trimmedQuery, utils.Config.Chain.ClConfig.SlotsPerEpoch) - } else if blsLikeRE.MatchString(query) { - searchQuery := ` - WHERE pubkey_text LIKE ($1 || '%') - OR deposit_address_text LIKE ($1 || '%')` - err = ReaderDb.Get(&count, fmt.Sprintf(blsQuery, joinQuery, searchQuery), - trimmedQuery) + OR bls.block_slot = $1 + OR bls.block_slot BETWEEN $1*$2 AND ($1+1)*$2-1` + err = ReaderDb.Get(&count, fmt.Sprintf(blsQuery, searchQuery, BlsChangeQueryLimit), + uiQuery, utils.Config.Chain.ClConfig.SlotsPerEpoch) } if err != nil { return 0, err @@ -2960,7 +3008,6 @@ func GetBLSChanges(query string, length, start uint64, orderBy, orderDir string) FROM blocks_bls_change bls INNER JOIN blocks b ON bls.block_root = b.blockroot AND b.status = '1' %s - %s ORDER BY bls.%s %s LIMIT $1 OFFSET $2` @@ -2969,40 +3016,28 @@ func GetBLSChanges(query string, length, start uint64, orderBy, orderDir string) var err error = nil if trimmedQuery != "" { - joinQuery := ` - LEFT JOIN ( - SELECT - validators.validatorindex as validatorindex, - eth1_deposits.from_address_text as deposit_address_text - FROM validators - INNER JOIN eth1_deposits ON validators.pubkey = eth1_deposits.publickey - ) AS val ON val.validatorindex = bls.validatorindex` - - // Check whether the query can be used for a validator, slot or epoch search - if uiQuery, parseErr := strconv.ParseUint(query, 10, 64); parseErr == nil { + if blsRE.MatchString(query) { + searchQuery := `WHERE bls.pubkey = $3` + pubkey, decErr := hex.DecodeString(trimmedQuery) + if decErr != nil { + return nil, decErr + } + err = ReaderDb.Select(&blsChange, fmt.Sprintf(blsQuery, searchQuery, orderBy, orderDir), + length, start, pubkey) + } else if uiQuery, parseErr := strconv.ParseUint(query, 10, 64); parseErr == nil { + // Check whether the query can be used for a validator, slot or epoch search searchQuery := ` WHERE bls.validatorindex = $3 - OR block_slot = $3 - OR (block_slot / $5) = $3 - OR pubkey_text LIKE ($4 || '%') - OR deposit_address_text LIKE ($4 || '%')` - - err = ReaderDb.Select(&blsChange, fmt.Sprintf(blsQuery, joinQuery, searchQuery, orderBy, orderDir), - length, start, uiQuery, trimmedQuery, utils.Config.Chain.ClConfig.SlotsPerEpoch) - } else if blsLikeRE.MatchString(query) { - searchQuery := ` - WHERE pubkey_text LIKE ($3 || '%') - OR deposit_address_text LIKE ($3 || '%')` - - err = ReaderDb.Select(&blsChange, fmt.Sprintf(blsQuery, joinQuery, searchQuery, orderBy, orderDir), - length, start, trimmedQuery) + OR bls.block_slot = $3 + OR bls.block_slot BETWEEN $3*$4 AND ($3+1)*$4-1` + err = ReaderDb.Select(&blsChange, fmt.Sprintf(blsQuery, searchQuery, orderBy, orderDir), + length, start, uiQuery, utils.Config.Chain.ClConfig.SlotsPerEpoch) } if err != nil { return nil, err } - } else { - err := ReaderDb.Select(&blsChange, fmt.Sprintf(blsQuery, "", "", orderBy, orderDir), length, start) + err := ReaderDb.Select(&blsChange, fmt.Sprintf(blsQuery, "", orderBy, orderDir), length, start) if err != nil { return nil, err } @@ -3160,16 +3195,20 @@ func GetPendingBLSChangeValidatorCount() (uint64, error) { } func GetLastExportedStatisticDay() (uint64, error) { - var lastStatsDay uint64 - err := ReaderDb.Get(&lastStatsDay, "SELECT COALESCE(MAX(day),0) FROM validator_stats_status WHERE status") + var lastStatsDay sql.NullInt64 + err := ReaderDb.Get(&lastStatsDay, "SELECT MAX(day) FROM validator_stats_status WHERE status") if err != nil { return 0, fmt.Errorf("error getting lastStatsDay %v", err) } - return lastStatsDay, nil + + if !lastStatsDay.Valid { + return 0, ErrNoStats + } + return uint64(lastStatsDay.Int64), nil } -func GetValidatorIncomePerforamance(validators []uint64, incomePerformance *types.ValidatorIncomePerformance) error { +func GetValidatorIncomePerformance(validators []uint64, incomePerformance *types.ValidatorIncomePerformance) error { validatorsPQArray := pq.Array(validators) // el rewards are converted from wei to gwei return ReaderDb.Get(incomePerformance, ` @@ -3179,7 +3218,6 @@ func GetValidatorIncomePerforamance(validators []uint64, incomePerformance *type COALESCE(SUM(cl_performance_31d), 0) AS cl_performance_31d, COALESCE(SUM(cl_performance_365d), 0) AS cl_performance_365d, COALESCE(SUM(cl_performance_total), 0) AS cl_performance_total, - COALESCE(SUM(cl_proposer_performance_total), 0) AS cl_proposer_performance_total, CAST(COALESCE(SUM(mev_performance_1d), 0) / 1e9 AS bigint) AS el_performance_1d, CAST(COALESCE(SUM(mev_performance_7d), 0) / 1e9 AS bigint) AS el_performance_7d, CAST(COALESCE(SUM(mev_performance_31d), 0) / 1e9 AS bigint) AS el_performance_31d, @@ -3249,10 +3287,24 @@ func GetValidatorBalanceForDay(validators []uint64, day uint64, balance *uint64) SELECT COALESCE(SUM(end_balance), 0) FROM validator_stats - WHERE day=$2 AND validatorindex = ANY($1) + WHERE validatorindex = ANY($1) AND day = $2 `, validatorsPQArray, day) } +func GetValidatorActivationBalance(validators []uint64, balance *uint64) error { + if len(validators) == 0 { + return fmt.Errorf("passing empty validator array is unsupported") + } + + validatorsPQArray := pq.Array(validators) + return ReaderDb.Get(balance, ` + SELECT + SUM(balanceactivation) + FROM validators + WHERE validatorindex = ANY($1) + `, validatorsPQArray) +} + func GetValidatorPropsosals(validators []uint64, proposals *[]types.ValidatorProposalInfo) error { validatorsPQArray := pq.Array(validators) @@ -3267,6 +3319,32 @@ func GetValidatorPropsosals(validators []uint64, proposals *[]types.ValidatorPro `, validatorsPQArray) } +func GetMissedSlots(slots []uint64) ([]uint64, error) { + slotsPQArray := pq.Array(slots) + missed := []uint64{} + + err := ReaderDb.Select(&missed, ` + SELECT + slot + FROM blocks + WHERE slot = ANY($1) AND status = '2' + `, slotsPQArray) + + return missed, err +} + +func GetMissedSlotsMap(slots []uint64) (map[uint64]bool, error) { + missedSlots, err := GetMissedSlots(slots) + if err != nil { + return nil, err + } + missedSlotsMap := make(map[uint64]bool, len(missedSlots)) + for _, slot := range missedSlots { + missedSlotsMap[slot] = true + } + return missedSlotsMap, nil +} + func GetOrphanedSlots(slots []uint64) ([]uint64, error) { slotsPQArray := pq.Array(slots) orphaned := []uint64{} @@ -3286,9 +3364,19 @@ func GetOrphanedSlotsMap(slots []uint64) (map[uint64]bool, error) { if err != nil { return nil, err } - orphanedSlotsMap := make(map[uint64]bool) + orphanedSlotsMap := make(map[uint64]bool, len(orphanedSlots)) for _, slot := range orphanedSlots { orphanedSlotsMap[slot] = true } return orphanedSlotsMap, nil } + +func GetBlockStatus(block int64, latestFinalizedEpoch uint64, epochInfo *types.EpochInfo) error { + return ReaderDb.Get(epochInfo, ` + SELECT (epochs.epoch <= $2) AS finalized, epochs.globalparticipationrate + FROM blocks + LEFT JOIN epochs ON blocks.epoch = epochs.epoch + WHERE blocks.exec_block_number = $1 + AND blocks.status='1'`, + block, latestFinalizedEpoch) +} diff --git a/db/ens.go b/db/ens.go index 556f36f4aa..4a7ddfb27f 100644 --- a/db/ens.go +++ b/db/ens.go @@ -106,7 +106,7 @@ func (bigtable *Bigtable) TransformEnsNameRegistered(blk *types.Eth1Block, cache } for _, lTopic := range log.GetTopics() { if isRegistarContract { - if bytes.Equal(lTopic, ens.NameRegisteredTopic) { + if bytes.Equal(lTopic, ens.NameRegisteredTopic) || bytes.Equal(lTopic, ens.NameRegisteredV2Topic) { foundNameIndex = j } else if bytes.Equal(lTopic, ens.NewResolverTopic) { foundResolverIndex = j @@ -163,13 +163,24 @@ func (bigtable *Bigtable) TransformEnsNameRegistered(blk *types.Eth1Block, cache Removed: log.GetRemoved(), } + var owner common.Address + var name string + nameRegistered, err := filterer.ParseNameRegistered(nameLog) if err != nil { - utils.LogError(err, fmt.Sprintf("indexing of register event failed parse register event at tx [%v] index [%v] on block [%v]", i, foundNameIndex, blk.Number), 0) - continue + nameRegisteredV2, err := filterer.ParseNameRegisteredV2(nameLog) + if err != nil { + utils.LogError(err, fmt.Sprintf("indexing of register event failed parse register event at tx [%v] index [%v] on block [%v]", i, foundNameIndex, blk.Number), 0) + continue + } + owner = nameRegisteredV2.Owner + name = nameRegisteredV2.Name + } else { + owner = nameRegistered.Owner + name = nameRegistered.Name } - if err = verifyName(nameRegistered.Name); err != nil { + if err = verifyName(name); err != nil { logger.Warnf("indexing of register event failed because of invalid name at tx [%v] index [%v] on block [%v]: %v", i, foundNameIndex, blk.Number, err) continue } @@ -181,9 +192,9 @@ func (bigtable *Bigtable) TransformEnsNameRegistered(blk *types.Eth1Block, cache } keys[fmt.Sprintf("%s:ENS:I:H:%x:%x", bigtable.chainId, resolver.Node, tx.GetHash())] = true - keys[fmt.Sprintf("%s:ENS:I:A:%x:%x", bigtable.chainId, nameRegistered.Owner, tx.GetHash())] = true - keys[fmt.Sprintf("%s:ENS:V:A:%x", bigtable.chainId, nameRegistered.Owner)] = true - keys[fmt.Sprintf("%s:ENS:V:N:%s", bigtable.chainId, nameRegistered.Name)] = true + keys[fmt.Sprintf("%s:ENS:I:A:%x:%x", bigtable.chainId, owner, tx.GetHash())] = true + keys[fmt.Sprintf("%s:ENS:V:A:%x", bigtable.chainId, owner)] = true + keys[fmt.Sprintf("%s:ENS:V:N:%s", bigtable.chainId, name)] = true } else if foundNameRenewedIndex > -1 { // We found a renew name event log := logs[foundNameRenewedIndex] @@ -297,11 +308,10 @@ func (bigtable *Bigtable) TransformEnsNameRegistered(blk *types.Eth1Block, cache return bulkData, bulkMetadataUpdates, nil } -// Ens names are only supported to a length of 40 -// Source: https://github.com/wealdtech/go-ens/blob/5b323a4ef0472f06c515723b060b166843b9db08/resolver.go#L190 func verifyName(name string) error { - if (strings.HasPrefix(name, "0x") && len(name) > 42) || (!strings.HasPrefix(name, "0x") && len(name) > 40) { - return fmt.Errorf("name too long") + // limited by max capacity of db (caused by btrees of indexes); tests showed maximum of 2684 (added buffer) + if len(name) > 2048 { + return fmt.Errorf("name too long: %v", name) } return nil } diff --git a/db/migrations/20230828105207_vlidator_performance_column_types.sql b/db/migrations/20230828105207_vlidator_performance_column_types.sql new file mode 100644 index 0000000000..44899c535e --- /dev/null +++ b/db/migrations/20230828105207_vlidator_performance_column_types.sql @@ -0,0 +1,21 @@ +-- +goose NO TRANSACTION +-- +goose Up +SELECT 'up SQL query - change validator performance columns to numeric'; +-- +goose StatementBegin +ALTER TABLE validator_performance + ALTER COLUMN el_performance_1d TYPE NUMERIC, + ALTER COLUMN el_performance_7d TYPE NUMERIC, + ALTER COLUMN el_performance_31d TYPE NUMERIC, + ALTER COLUMN el_performance_365d TYPE NUMERIC, + ALTER COLUMN el_performance_total TYPE NUMERIC, + ALTER COLUMN mev_performance_1d TYPE NUMERIC, + ALTER COLUMN mev_performance_7d TYPE NUMERIC, + ALTER COLUMN mev_performance_31d TYPE NUMERIC, + ALTER COLUMN mev_performance_365d TYPE NUMERIC, + ALTER COLUMN mev_performance_total TYPE NUMERIC; +-- +goose StatementEnd + +-- +goose Down +SELECT 'down SQL query - we do not revert the validator performance columns to BIGINT as this could cause an out of range error'; +-- +goose StatementBegin +-- +goose StatementEnd diff --git a/db/migrations/20230828113508_add_chain_head_table.sql b/db/migrations/20230828113508_add_chain_head_table.sql new file mode 100644 index 0000000000..8e0158b911 --- /dev/null +++ b/db/migrations/20230828113508_add_chain_head_table.sql @@ -0,0 +1,26 @@ +-- +goose Up +-- +goose StatementBegin +SELECT 'create chain_head table'; +CREATE TABLE IF NOT EXISTS + chain_head ( + finalized_block_root bytea, + finalized_epoch INT, + finalized_slot INT, + head_block_root bytea, + head_epoch INT, + head_slot INT, + justified_block_root bytea, + justified_epoch INT, + justified_slot INT, + previous_justified_block_root bytea, + previous_justified_epoch INT, + previous_justified_slot INT, + PRIMARY KEY (finalized_epoch) + ); +-- +goose StatementEnd + +-- +goose Down +-- +goose StatementBegin +SELECT 'delete chain_head table'; +DROP TABLE IF EXISTS chain_head; +-- +goose StatementEnd \ No newline at end of file diff --git a/db/migrations/20230828130109_rem_proposer_rewards_statistic_columns.sql b/db/migrations/20230828130109_rem_proposer_rewards_statistic_columns.sql new file mode 100644 index 0000000000..c67f22612c --- /dev/null +++ b/db/migrations/20230828130109_rem_proposer_rewards_statistic_columns.sql @@ -0,0 +1,15 @@ +-- +goose Up +-- +goose StatementBegin +SELECT 'up SQL query remove cl_proposer columns'; +ALTER TABLE validator_stats DROP COLUMN IF EXISTS cl_proposer_rewards_gwei; +ALTER TABLE validator_stats DROP COLUMN IF EXISTS cl_proposer_rewards_gwei_total; +ALTER TABLE validator_performance DROP COLUMN IF EXISTS cl_proposer_performance_total; +-- +goose StatementEnd + +-- +goose Down +-- +goose StatementBegin +SELECT 'down SQL query add cl_proposer columns'; +ALTER TABLE validator_stats ADD COLUMN IF NOT EXISTS cl_proposer_rewards_gwei BIGINT; +ALTER TABLE validator_stats ADD COLUMN IF NOT EXISTS cl_proposer_rewards_gwei_total BIGINT; +ALTER TABLE validator_performance ADD COLUMN IF NOT EXISTS cl_proposer_performance_total BIGINT; +-- +goose StatementEnd diff --git a/db/migrations/20230830144100_improve_blocks_bls_change_pubkey_text_idx.sql b/db/migrations/20230830144100_improve_blocks_bls_change_pubkey_text_idx.sql new file mode 100644 index 0000000000..535a9e3689 --- /dev/null +++ b/db/migrations/20230830144100_improve_blocks_bls_change_pubkey_text_idx.sql @@ -0,0 +1,40 @@ +-- +goose NO TRANSACTION + +-- +goose Up + +-- +goose StatementBegin +DROP INDEX CONCURRENTLY IF EXISTS idx_blocks_bls_change_pubkey_text; +-- +goose StatementEnd +-- +goose StatementBegin +ALTER TABLE blocks_bls_change DROP COLUMN IF EXISTS pubkey_text; +-- +goose StatementEnd +-- +goose StatementBegin +DROP INDEX CONCURRENTLY IF EXISTS idx_blocks_withdrawals_search; +-- +goose StatementEnd +-- +goose StatementBegin +DROP INDEX CONCURRENTLY IF EXISTS idx_blocks_withdrawals_address_text; +-- +goose StatementEnd +-- +goose StatementBegin +ALTER TABLE blocks_withdrawals DROP COLUMN IF EXISTS address_text; +-- +goose StatementEnd +-- +goose StatementBegin +CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_blocks_withdrawals_address ON blocks_withdrawals (address); +-- +goose StatementEnd + +-- +goose Down + +-- +goose StatementBegin +ALTER TABLE blocks_withdrawals ADD COLUMN IF NOT EXISTS address_text TEXT NOT NULL DEFAULT ''; +-- +goose StatementEnd +-- +goose StatementBegin +CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_blocks_withdrawals_address_text ON blocks_withdrawals USING gin (address_text gin_trgm_ops); +-- +goose StatementEnd +-- +goose StatementBegin +ALTER TABLE blocks_bls_change ADD COLUMN IF NOT EXISTS pubkey_text TEXT NOT NULL DEFAULT ''; +-- +goose StatementEnd +-- +goose StatementBegin +CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_blocks_bls_change_pubkey_text ON blocks_bls_change USING gin (pubkey_text gin_trgm_ops); +-- +goose StatementEnd +-- +goose StatementBegin +CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_blocks_bls_change_search ON blocks_bls_change (validatorindex, block_slot, pubkey_text); +-- +goose StatementEnd diff --git a/db/migrations/20230904092500_remove_streakleaderboard_table.sql b/db/migrations/20230904092500_remove_streakleaderboard_table.sql new file mode 100644 index 0000000000..efc351c800 --- /dev/null +++ b/db/migrations/20230904092500_remove_streakleaderboard_table.sql @@ -0,0 +1,39 @@ +-- +goose Up +-- +goose StatementBegin +SELECT 'up SQL query - remove table validator_attestation_streaks'; +DROP TABLE IF EXISTS validator_attestation_streaks CASCADE; +-- +goose StatementEnd + +-- +goose Down +-- +goose StatementBegin +SELECT 'down SQL query - add table validator_attestation_streaks and indexes'; +CREATE TABLE IF NOT EXISTS + validator_attestation_streaks ( + validatorindex INT NOT NULL, + status INT NOT NULL, + START INT NOT NULL, + LENGTH INT NOT NULL, + longest BOOLEAN NOT NULL, + CURRENT BOOLEAN NOT NULL, + PRIMARY KEY (validatorindex, status, START) + ); +-- +goose StatementEnd + +-- +goose StatementBegin +CREATE INDEX IF NOT EXISTS idx_validator_attestation_streaks_validatorindex ON validator_attestation_streaks (validatorindex); +-- +goose StatementEnd +-- +goose StatementBegin +CREATE INDEX IF NOT EXISTS idx_validator_attestation_streaks_status ON validator_attestation_streaks (status); +-- +goose StatementEnd +-- +goose StatementBegin +CREATE INDEX IF NOT EXISTS idx_validator_attestation_streaks_length ON validator_attestation_streaks (LENGTH); +-- +goose StatementEnd +-- +goose StatementBegin +CREATE INDEX IF NOT EXISTS idx_validator_attestation_streaks_start ON validator_attestation_streaks (START); +-- +goose StatementEnd +-- +goose StatementBegin +CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_validator_attestation_streaks_status_longest ON public.validator_attestation_streaks USING btree (status, longest); +-- +goose StatementEnd +-- +goose StatementBegin +CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_validator_attestation_streaks_status_current ON public.validator_attestation_streaks USING btree (status, current); +-- +goose StatementEnd diff --git a/db/migrations/20230912091400_remove_chain_head_table.sql b/db/migrations/20230912091400_remove_chain_head_table.sql new file mode 100644 index 0000000000..a9658f7f45 --- /dev/null +++ b/db/migrations/20230912091400_remove_chain_head_table.sql @@ -0,0 +1,26 @@ +-- +goose Up +-- +goose StatementBegin +SELECT 'delete chain_head table'; +DROP TABLE IF EXISTS chain_head; +-- +goose StatementEnd + +-- +goose Down +-- +goose StatementBegin +SELECT 'create chain_head table'; +CREATE TABLE IF NOT EXISTS + chain_head ( + finalized_block_root bytea, + finalized_epoch INT, + finalized_slot INT, + head_block_root bytea, + head_epoch INT, + head_slot INT, + justified_block_root bytea, + justified_epoch INT, + justified_slot INT, + previous_justified_block_root bytea, + previous_justified_epoch INT, + previous_justified_slot INT, + PRIMARY KEY (finalized_epoch) + ); +-- +goose StatementEnd diff --git a/db/node_jobs.go b/db/node_jobs.go index ea01189d08..81da90d593 100644 --- a/db/node_jobs.go +++ b/db/node_jobs.go @@ -6,7 +6,7 @@ import ( "eth2-exporter/types" "eth2-exporter/utils" "fmt" - "io/ioutil" + "io" "net/http" "strings" "time" @@ -311,7 +311,7 @@ func SubmitBLSToExecutionChangesNodeJob(job *types.NodeJob) error { } jobStatus := types.SubmittedToNodeNodeJobStatus if resp.StatusCode != 200 { - d, _ := ioutil.ReadAll(resp.Body) + d, _ := io.ReadAll(resp.Body) if len(d) > 1000 { d = d[:1000] } @@ -447,7 +447,7 @@ func SubmitVoluntaryExitNodeJob(job *types.NodeJob) error { } jobStatus := types.SubmittedToNodeNodeJobStatus if resp.StatusCode != 200 { - d, _ := ioutil.ReadAll(resp.Body) + d, _ := io.ReadAll(resp.Body) if len(d) > 1000 { d = d[:1000] } diff --git a/db/statistics.go b/db/statistics.go index 554d3b6cb8..9f0cff47bf 100644 --- a/db/statistics.go +++ b/db/statistics.go @@ -11,10 +11,10 @@ import ( "fmt" "math/big" "strings" - "sync" "time" "github.com/ethereum/go-ethereum/common" + "github.com/jmoiron/sqlx" "github.com/lib/pq" "github.com/shopspring/decimal" "github.com/sirupsen/logrus" @@ -52,7 +52,13 @@ func WriteValidatorStatisticsForDay(day uint64, concurrencyTotal uint64, concurr } exported := Exported{} - err := ReaderDb.Get(&exported, ` + tx, err := WriterDb.Beginx() + if err != nil { + return fmt.Errorf("error starting transaction: %w", err) + } + defer tx.Rollback() + + err = tx.Get(&exported, ` SELECT status, failed_attestations_exported, @@ -73,6 +79,17 @@ func WriteValidatorStatisticsForDay(day uint64, concurrencyTotal uint64, concurr } logger.Infof("getting exported state took %v", time.Since(start)) + maxValidatorIndex, err := BigtableClient.GetMaxValidatorindexForEpoch(lastEpoch) + if err != nil { + return err + } + validators := make([]uint64, 0, maxValidatorIndex) + + logger.Infof("processing statistics for validators 0-%d", maxValidatorIndex) + for i := uint64(0); i <= maxValidatorIndex; i++ { + validators = append(validators, i) + } + if exported.FailedAttestations && exported.SyncDuties && exported.WithdrawalsDeposits && exported.Balance && exported.ClRewards && exported.ElRewards && exported.TotalAccumulation && exported.TotalPerformance && exported.BlockStats && exported.Status { logger.Infof("Skipping day %v as it is already exported", day) return nil @@ -80,77 +97,77 @@ func WriteValidatorStatisticsForDay(day uint64, concurrencyTotal uint64, concurr if exported.FailedAttestations { logger.Infof("Skipping failed attestations") - } else if err := WriteValidatorFailedAttestationsStatisticsForDay(day, concurrencyFailedAttestations); err != nil { + } else if err := WriteValidatorFailedAttestationsStatisticsForDay(validators, day, concurrencyFailedAttestations, tx); err != nil { return fmt.Errorf("error in WriteValidatorFailedAttestationsStatisticsForDay: %w", err) } if exported.SyncDuties { logger.Infof("Skipping sync duties") - } else if err := WriteValidatorSyncDutiesForDay(day); err != nil { + } else if err := WriteValidatorSyncDutiesForDay(validators, day, tx); err != nil { return fmt.Errorf("error in WriteValidatorSyncDutiesForDay: %w", err) } if exported.WithdrawalsDeposits { logger.Infof("Skipping withdrawals / deposits") - } else if err := WriteValidatorDepositWithdrawals(day); err != nil { + } else if err := WriteValidatorDepositWithdrawals(day, tx); err != nil { return fmt.Errorf("error in WriteValidatorDepositWithdrawals: %w", err) } if exported.BlockStats { logger.Infof("Skipping block stats") - } else if err := WriteValidatorBlockStats(day); err != nil { + } else if err := WriteValidatorBlockStats(day, tx); err != nil { return fmt.Errorf("error in WriteValidatorBlockStats: %w", err) } if exported.Balance { logger.Infof("Skipping balances") - } else if err := WriteValidatorBalances(day); err != nil { + } else if err := WriteValidatorBalances(validators, day, tx); err != nil { return fmt.Errorf("error in WriteValidatorBalances: %w", err) } if exported.ClRewards { logger.Infof("Skipping cl rewards") - } else if err := WriteValidatorClIcome(day, concurrencyCl); err != nil { + } else if err := WriteValidatorClIcome(validators, day, concurrencyCl, tx); err != nil { return fmt.Errorf("error in WriteValidatorClIcome: %w", err) } if exported.ElRewards { logger.Infof("Skipping el rewards") - } else if err := WriteValidatorElIcome(day); err != nil { + } else if err := WriteValidatorElIcome(day, tx); err != nil { return fmt.Errorf("error in WriteValidatorElIcome: %w", err) } if exported.TotalAccumulation { logger.Infof("Skipping total accumulation") - } else if err := WriteValidatorTotalAccumulation(day, concurrencyTotal); err != nil { + } else if err := WriteValidatorTotalAccumulation(day, concurrencyTotal, tx); err != nil { return fmt.Errorf("error in WriteValidatorTotalAccumulation: %w", err) } if exported.TotalPerformance { logger.Infof("Skipping total performance") - } else if err := WriteValidatorTotalPerformance(day, concurrencyTotal); err != nil { + } else if err := WriteValidatorTotalPerformance(day, concurrencyTotal, tx); err != nil { return fmt.Errorf("error in WriteValidatorTotalPerformance: %w", err) } - if err := WriteValidatorStatsExported(day); err != nil { + if err := WriteValidatorStatsExported(day, tx); err != nil { return fmt.Errorf("error in WriteValidatorStatsExported: %w", err) } + err = tx.Commit() + + if err != nil { + return fmt.Errorf("error committing tx: %w", err) + } logger.Infof("statistics export of day %v completed, took %v", day, time.Since(exportStart)) return nil } -func WriteValidatorStatsExported(day uint64) error { - tx, err := WriterDb.Beginx() - if err != nil { - return err - } - defer tx.Rollback() +func WriteValidatorStatsExported(day uint64, tx *sqlx.Tx) error { start := time.Now() logger.Infof("marking day export as completed in the validator_stats_status table for day %v", day) - _, err = tx.Exec(` + _, err := tx.Exec(` UPDATE validator_stats_status SET status = true WHERE day=$1 @@ -169,14 +186,10 @@ func WriteValidatorStatsExported(day uint64) error { } logger.Infof("marking completed, took %v", time.Since(start)) - err = tx.Commit() - if err != nil { - return err - } return nil } -func WriteValidatorTotalAccumulation(day uint64, concurrency uint64) error { +func WriteValidatorTotalAccumulation(day uint64, concurrency uint64, tx *sqlx.Tx) error { ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(time.Minute*10)) defer cancel() exportStart := time.Now() @@ -198,7 +211,7 @@ func WriteValidatorTotalAccumulation(day uint64, concurrency uint64) error { CurrentFailedAttestations bool `db:"cur_failed_attestations_exported"` } exported := Exported{} - err := ReaderDb.Get(&exported, ` + err := tx.Get(&exported, ` SELECT last.total_accumulation_exported as last_total_accumulation_exported, cur.cl_rewards_exported as cur_cl_rewards_exported, @@ -241,12 +254,11 @@ func WriteValidatorTotalAccumulation(day uint64, concurrency uint64) error { return gCtx.Err() default: } - _, err = WriterDb.Exec(`INSERT INTO validator_stats ( + _, err = tx.Exec(`INSERT INTO validator_stats ( validatorindex, day, cl_rewards_gwei_total, - cl_proposer_rewards_gwei_total, el_rewards_wei_total, mev_rewards_wei_total, @@ -260,7 +272,6 @@ func WriteValidatorTotalAccumulation(day uint64, concurrency uint64) error { vs1.validatorindex, vs1.day, COALESCE(vs1.cl_rewards_gwei, 0) + COALESCE(vs2.cl_rewards_gwei_total, 0), - COALESCE(vs1.cl_proposer_rewards_gwei, 0) + COALESCE(vs2.cl_proposer_rewards_gwei_total, 0), COALESCE(vs1.el_rewards_wei, 0) + COALESCE(vs2.el_rewards_wei_total, 0), COALESCE(vs1.mev_rewards_wei, 0) + COALESCE(vs2.mev_rewards_wei_total, 0), COALESCE(vs1.missed_attestations, 0) + COALESCE(vs2.missed_attestations_total, 0), @@ -271,7 +282,6 @@ func WriteValidatorTotalAccumulation(day uint64, concurrency uint64) error { ) ON CONFLICT (validatorindex, day) DO UPDATE SET cl_rewards_gwei_total = excluded.cl_rewards_gwei_total, - cl_proposer_rewards_gwei_total = excluded.cl_proposer_rewards_gwei_total, el_rewards_wei_total = excluded.el_rewards_wei_total, mev_rewards_wei_total = excluded.mev_rewards_wei_total, missed_attestations_total = excluded.missed_attestations_total, @@ -293,7 +303,7 @@ func WriteValidatorTotalAccumulation(day uint64, concurrency uint64) error { } logger.Infof("export completed, took %v", time.Since(start)) - if err = markColumnExported(day, "total_accumulation_exported"); err != nil { + if err = markColumnExported(day, "total_accumulation_exported", tx); err != nil { return err } @@ -301,7 +311,7 @@ func WriteValidatorTotalAccumulation(day uint64, concurrency uint64) error { return nil } -func WriteValidatorTotalPerformance(day uint64, concurrency uint64) error { +func WriteValidatorTotalPerformance(day uint64, concurrency uint64, tx *sqlx.Tx) error { ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(time.Minute*10)) defer cancel() exportStart := time.Now() @@ -320,7 +330,7 @@ func WriteValidatorTotalPerformance(day uint64, concurrency uint64) error { CurrentTotalAccumulation bool `db:"cur_total_accumulation_exported"` } exported := Exported{} - err := ReaderDb.Get(&exported, ` + err := tx.Get(&exported, ` SELECT last.total_performance_exported as last_total_performance_exported, cur.total_accumulation_exported as cur_total_accumulation_exported @@ -361,7 +371,7 @@ func WriteValidatorTotalPerformance(day uint64, concurrency uint64) error { default: } - _, err = WriterDb.Exec(`insert into validator_performance ( + _, err = tx.Exec(`insert into validator_performance ( validatorindex, balance, rank7d, @@ -371,7 +381,6 @@ func WriteValidatorTotalPerformance(day uint64, concurrency uint64) error { cl_performance_31d, cl_performance_365d, cl_performance_total, - cl_proposer_performance_total, el_performance_1d, el_performance_7d, @@ -395,7 +404,6 @@ func WriteValidatorTotalPerformance(day uint64, concurrency uint64) error { coalesce(vs_now.cl_rewards_gwei_total, 0) - coalesce(vs_31d.cl_rewards_gwei_total, 0) as cl_performance_31d, coalesce(vs_now.cl_rewards_gwei_total, 0) - coalesce(vs_365d.cl_rewards_gwei_total, 0) as cl_performance_365d, coalesce(vs_now.cl_rewards_gwei_total, 0) as cl_performance_total, - coalesce(vs_now.cl_proposer_rewards_gwei_total, 0) as cl_proposer_performance_total, coalesce(vs_now.el_rewards_wei_total, 0) - coalesce(vs_1d.el_rewards_wei_total, 0) as el_performance_1d, coalesce(vs_now.el_rewards_wei_total, 0) - coalesce(vs_7d.el_rewards_wei_total, 0) as el_performance_7d, @@ -424,7 +432,6 @@ func WriteValidatorTotalPerformance(day uint64, concurrency uint64) error { cl_performance_31d=excluded.cl_performance_31d, cl_performance_365d=excluded.cl_performance_365d, cl_performance_total=excluded.cl_performance_total, - cl_proposer_performance_total=excluded.cl_proposer_performance_total, el_performance_1d=excluded.el_performance_1d, el_performance_7d=excluded.el_performance_7d, @@ -456,7 +463,7 @@ func WriteValidatorTotalPerformance(day uint64, concurrency uint64) error { start = time.Now() logger.Infof("populate validator_performance rank7d") - _, err = WriterDb.Exec(` + _, err = tx.Exec(` WITH ranked_performance AS ( SELECT validatorindex, @@ -474,7 +481,7 @@ func WriteValidatorTotalPerformance(day uint64, concurrency uint64) error { logger.Infof("export completed, took %v", time.Since(start)) - if err = markColumnExported(day, "total_performance_exported"); err != nil { + if err = markColumnExported(day, "total_performance_exported", tx); err != nil { return err } @@ -482,7 +489,7 @@ func WriteValidatorTotalPerformance(day uint64, concurrency uint64) error { return nil } -func WriteValidatorBlockStats(day uint64) error { +func WriteValidatorBlockStats(day uint64, tx *sqlx.Tx) error { exportStart := time.Now() defer func() { metrics.TaskDuration.WithLabelValues("db_update_validator_block_stats").Observe(time.Since(exportStart).Seconds()) @@ -492,15 +499,22 @@ func WriteValidatorBlockStats(day uint64) error { return err } - firstEpoch, lastEpoch := utils.GetFirstAndLastEpochForDay(day) - - tx, err := WriterDb.Beginx() + start := time.Now() + resetQry := ` + UPDATE validator_stats SET + proposed_blocks = NULL, + missed_blocks = NULL, + orphaned_blocks = NULL + WHERE day = $1;` + _, err := tx.Exec(resetQry, day) if err != nil { - return err + return fmt.Errorf("error resetting proposer duty validator_stats for day [%v]: %w", day, err) } - defer tx.Rollback() + logger.Infof("proposer duty reset completed, took %v", time.Since(start)) - start := time.Now() + firstEpoch, lastEpoch := utils.GetFirstAndLastEpochForDay(day) + + start = time.Now() logger.Infof("exporting proposed_blocks, missed_blocks and orphaned_blocks statistics") _, err = tx.Exec(` @@ -533,13 +547,9 @@ func WriteValidatorBlockStats(day uint64) error { if err != nil { return fmt.Errorf("error inserting slashings into validator_stats for day [%v], firstEpoch [%v] and lastEpoch [%v]: %w", day, firstEpoch, lastEpoch, err) } - - if err = tx.Commit(); err != nil { - return err - } logger.Infof("export completed, took %v", time.Since(start)) - if err = markColumnExported(day, "block_stats_exported"); err != nil { + if err = markColumnExported(day, "block_stats_exported", tx); err != nil { return err } @@ -547,7 +557,7 @@ func WriteValidatorBlockStats(day uint64) error { return nil } -func WriteValidatorElIcome(day uint64) error { +func WriteValidatorElIcome(day uint64, tx *sqlx.Tx) error { exportStart := time.Now() defer func() { metrics.TaskDuration.WithLabelValues("db_update_validator_el_income_stats").Observe(time.Since(exportStart).Seconds()) @@ -557,15 +567,21 @@ func WriteValidatorElIcome(day uint64) error { return err } - firstEpoch, lastEpoch := utils.GetFirstAndLastEpochForDay(day) - - tx, err := WriterDb.Beginx() + start := time.Now() + resetQry := ` + UPDATE validator_stats SET + el_rewards_wei = NULL, + mev_rewards_wei = NULL + WHERE day = $1;` + _, err := tx.Exec(resetQry, day) if err != nil { - return err + return fmt.Errorf("error resetting el income validator_stats for day [%v]: %w", day, err) } - defer tx.Rollback() + logger.Infof("el income reset completed, took %v", time.Since(start)) - start := time.Now() + firstEpoch, lastEpoch := utils.GetFirstAndLastEpochForDay(day) + + start = time.Now() logger.Infof("exporting mev & el rewards") @@ -652,12 +668,9 @@ func WriteValidatorElIcome(day uint64) error { } } - if err = tx.Commit(); err != nil { - return err - } logger.Infof("export completed, took %v", time.Since(start)) - if err = markColumnExported(day, "el_rewards_exported"); err != nil { + if err = markColumnExported(day, "el_rewards_exported", tx); err != nil { return err } @@ -665,7 +678,7 @@ func WriteValidatorElIcome(day uint64) error { return nil } -func WriteValidatorClIcome(day uint64, concurrency uint64) error { +func WriteValidatorClIcome(validators []uint64, day uint64, concurrency uint64, tx *sqlx.Tx) error { ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(time.Minute*10)) defer cancel() exportStart := time.Now() @@ -678,6 +691,17 @@ func WriteValidatorClIcome(day uint64, concurrency uint64) error { } start := time.Now() + resetQry := ` + UPDATE validator_stats SET + cl_rewards_gwei = NULL + WHERE day = $1;` + _, err := tx.Exec(resetQry, day) + if err != nil { + return fmt.Errorf("error resetting cl income validator_stats for day [%v]: %w", day, err) + } + logger.Infof("cl income reset completed, took %v", time.Since(start)) + + start = time.Now() logger.Infof("validating if required data has been exported for cl rewards") type Exported struct { LastBalanceExported bool `db:"last_balance_exported"` @@ -685,7 +709,7 @@ func WriteValidatorClIcome(day uint64, concurrency uint64) error { CurrentWithdrawalsDepositsExported bool `db:"cur_withdrawals_deposits_exported"` } exported := Exported{} - err := ReaderDb.Get(&exported, ` + err = tx.Get(&exported, ` SELECT last.balance_exported as last_balance_exported, cur.balance_exported as cur_balance_exported, cur.withdrawals_deposits_exported as cur_withdrawals_deposits_exported FROM validator_stats_status cur INNER JOIN validator_stats_status last @@ -701,29 +725,22 @@ func WriteValidatorClIcome(day uint64, concurrency uint64) error { logger.Infof("validating took %v", time.Since(start)) start = time.Now() - firstEpoch, lastEpoch := utils.GetFirstAndLastEpochForDay(day) + _, lastEpoch := utils.GetFirstAndLastEpochForDay(day) logger.Infof("exporting cl_rewards_wei statistics") - incomeStats, err := BigtableClient.GetAggregatedValidatorIncomeDetailsHistory([]uint64{}, firstEpoch, lastEpoch) + + maxValidatorIndex, err := BigtableClient.GetMaxValidatorindexForEpoch(lastEpoch) if err != nil { - return fmt.Errorf("error in GetAggregatedValidatorIncomeDetailsHistory for firstEpoch [%v] and lastEpoch [%v]: %w", firstEpoch, lastEpoch, err) + return fmt.Errorf("error in GetAggregatedValidatorIncomeDetailsHistory: could not get max validator index from validator income history for last epoch [%v] of day [%v]: %v", lastEpoch, day, err) + } else if maxValidatorIndex == uint64(0) { + return fmt.Errorf("error in GetAggregatedValidatorIncomeDetailsHistory: no validator found for last epoch [%v] of day [%v]: %v", lastEpoch, day, err) } - logrus.Infof("getting cl income done in %v, now we export them to the db", time.Since(start)) - start = time.Now() - - maxValidatorIndex := uint64(0) - for validator := range incomeStats { - if validator > maxValidatorIndex { - maxValidatorIndex = validator - } - } maxValidatorIndex++ g, gCtx := errgroup.WithContext(ctx) g.SetLimit(int(concurrency)) - numArgs := 3 batchSize := 100 // max parameters: 65535 / 3, but it's faster in smaller batches for b := 0; b < int(maxValidatorIndex); b += batchSize { start := b @@ -732,38 +749,13 @@ func WriteValidatorClIcome(day uint64, concurrency uint64) error { end = int(maxValidatorIndex) } - logrus.Info(start, end) - valueStrings := make([]string, 0, batchSize) - valueArgs := make([]interface{}, 0, batchSize*numArgs) - for i := start; i < end; i++ { - clProposerRewards := uint64(0) - - if incomeStats[uint64(i)] != nil { - clProposerRewards = incomeStats[uint64(i)].ProposerAttestationInclusionReward + incomeStats[uint64(i)].ProposerSlashingInclusionReward + incomeStats[uint64(i)].ProposerSyncInclusionReward - } - valueStrings = append(valueStrings, fmt.Sprintf("($%d, $%d, $%d)", (i-start)*numArgs+1, (i-start)*numArgs+2, (i-start)*numArgs+3)) - valueArgs = append(valueArgs, i) - valueArgs = append(valueArgs, day) - valueArgs = append(valueArgs, clProposerRewards) - } - stmt := fmt.Sprintf(` - insert into validator_stats (validatorindex, day, cl_proposer_rewards_gwei) VALUES - %s - on conflict (validatorindex, day) do update set cl_proposer_rewards_gwei = excluded.cl_proposer_rewards_gwei;`, - strings.Join(valueStrings, ",")) - g.Go(func() error { select { case <-gCtx.Done(): return gCtx.Err() default: } - _, err := WriterDb.Exec(stmt, valueArgs...) - if err != nil { - return fmt.Errorf("error inserting cl_proposer_rewards_gwei into validator_stats for day [%v], start [%v] and end [%v]: %w", day, start, end, err) - } - logrus.Infof("saving validator proposer rewards gwei batch %v completed", start) - stmt = ` + stmt := ` INSERT INTO validator_stats (validatorindex, day, cl_rewards_gwei) ( SELECT cur.validatorindex, cur.day, COALESCE(cur.end_balance, 0) - COALESCE(last.end_balance, 0) + COALESCE(cur.withdrawals_amount, 0) - COALESCE(cur.deposits_amount, 0) AS cl_rewards_gwei @@ -785,7 +777,7 @@ func WriteValidatorClIcome(day uint64, concurrency uint64) error { ON CONFLICT (validatorindex, day) DO UPDATE SET cl_rewards_gwei = excluded.cl_rewards_gwei;` } - _, err = WriterDb.Exec(stmt, day, start, end) + _, err = tx.Exec(stmt, day, start, end) if err != nil { return fmt.Errorf("error inserting cl_rewards_gwei into validator_stats for day [%v], start [%v] and end [%v]: %w", day, start, end, err) } @@ -801,7 +793,7 @@ func WriteValidatorClIcome(day uint64, concurrency uint64) error { logger.Infof("export completed, took %v", time.Since(start)) - if err = markColumnExported(day, "cl_rewards_exported"); err != nil { + if err = markColumnExported(day, "cl_rewards_exported", tx); err != nil { return err } @@ -809,7 +801,7 @@ func WriteValidatorClIcome(day uint64, concurrency uint64) error { return nil } -func WriteValidatorBalances(day uint64) error { +func WriteValidatorBalances(validators []uint64, day uint64, tx *sqlx.Tx) error { ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(time.Minute*10)) defer cancel() @@ -822,12 +814,30 @@ func WriteValidatorBalances(day uint64) error { return err } + start := time.Now() + resetQry := ` + UPDATE validator_stats SET + min_balance = NULL, + max_balance = NULL, + min_effective_balance = NULL, + max_effective_balance = NULL, + start_balance = NULL, + start_effective_balance = NULL, + end_balance = NULL, + end_effective_balance = NULL + WHERE day = $1;` + _, err := tx.Exec(resetQry, day) + if err != nil { + return fmt.Errorf("error resetting balances validator_stats for day [%v]: %w", day, err) + } + logger.Infof("balances reset completed, took %v", time.Since(start)) + firstEpoch, lastEpoch := utils.GetFirstAndLastEpochForDay(day) - start := time.Now() + start = time.Now() logger.Infof("exporting min_balance, max_balance, min_effective_balance, max_effective_balance, start_balance, start_effective_balance, end_balance and end_effective_balance statistics") - balanceStatistics, err := BigtableClient.GetValidatorBalanceStatistics(firstEpoch, lastEpoch) + balanceStatistics, err := BigtableClient.GetValidatorBalanceStatistics(validators, firstEpoch, lastEpoch) if err != nil { return fmt.Errorf("error in GetValidatorBalanceStatistics for firstEpoch [%v] and lastEpoch [%v]: %w", firstEpoch, lastEpoch, err) } @@ -878,7 +888,7 @@ func WriteValidatorBalances(day uint64) error { %s on conflict (validatorindex, day) do update set min_balance = excluded.min_balance, max_balance = excluded.max_balance, min_effective_balance = excluded.min_effective_balance, max_effective_balance = excluded.max_effective_balance, start_balance = excluded.start_balance, start_effective_balance = excluded.start_effective_balance, end_balance = excluded.end_balance, end_effective_balance = excluded.end_effective_balance;`, strings.Join(valueStrings, ",")) - _, err := WriterDb.Exec(stmt, valueArgs...) + _, err := tx.Exec(stmt, valueArgs...) if err != nil { return fmt.Errorf("error inserting balances into validator_stats for day [%v], start [%v] and end [%v]: %w", day, start, end, err) @@ -895,7 +905,7 @@ func WriteValidatorBalances(day uint64) error { logger.Infof("export completed, took %v", time.Since(start)) - if err = markColumnExported(day, "balance_exported"); err != nil { + if err = markColumnExported(day, "balance_exported", tx); err != nil { return err } @@ -903,7 +913,7 @@ func WriteValidatorBalances(day uint64) error { return nil } -func WriteValidatorDepositWithdrawals(day uint64) error { +func WriteValidatorDepositWithdrawals(day uint64, tx *sqlx.Tx) error { exportStart := time.Now() defer func() { metrics.TaskDuration.WithLabelValues("db_update_validator_deposit_withdrawal_stats").Observe(time.Since(exportStart).Seconds()) @@ -922,12 +932,6 @@ func WriteValidatorDepositWithdrawals(day uint64) error { } lastSlot := utils.GetLastBalanceInfoSlotForDay(day) - tx, err := WriterDb.Beginx() - if err != nil { - return err - } - defer tx.Rollback() - start := time.Now() logrus.Infof("Resetting Withdrawals + Deposits for day [%v]", day) @@ -945,11 +949,11 @@ func WriteValidatorDepositWithdrawals(day uint64) error { withdrawals_amount = NULL WHERE day = $1%s;`, firstDayExtraCondition) - _, err = tx.Exec(resetQry, day) + _, err := tx.Exec(resetQry, day) if err != nil { - return fmt.Errorf("error resetting validator_stats for day [%v]: %w", day, err) + return fmt.Errorf("error resetting deposit & withdrawal validator_stats for day [%v]: %w", day, err) } - logger.Infof("reset completed, took %v", time.Since(start)) + logger.Infof("deposit & withdrawal reset completed, took %v", time.Since(start)) start = time.Now() logrus.Infof("Update Withdrawals + Deposits for day [%v] slot %v -> %v", day, firstSlot, lastSlot) @@ -1011,13 +1015,10 @@ func WriteValidatorDepositWithdrawals(day uint64) error { if err != nil { return fmt.Errorf("error inserting withdrawals into validator_stats for day [%v], firstSlot [%v] and lastSlot [%v]: %w", day, firstSlot, lastSlot, err) } - if err = tx.Commit(); err != nil { - return err - } logger.Infof("export completed, took %v", time.Since(start)) - if err = markColumnExported(day, "withdrawals_deposits_exported"); err != nil { + if err = markColumnExported(day, "withdrawals_deposits_exported", tx); err != nil { return err } @@ -1025,7 +1026,7 @@ func WriteValidatorDepositWithdrawals(day uint64) error { return nil } -func WriteValidatorSyncDutiesForDay(day uint64) error { +func WriteValidatorSyncDutiesForDay(validators []uint64, day uint64, tx *sqlx.Tx) error { exportStart := time.Now() defer func() { metrics.TaskDuration.WithLabelValues("db_update_validator_sync_stats").Observe(time.Since(exportStart).Seconds()) @@ -1035,12 +1036,31 @@ func WriteValidatorSyncDutiesForDay(day uint64) error { return err } + start := time.Now() + resetQry := ` + UPDATE validator_stats SET + participated_sync = NULL, + missed_sync = NULL, + orphaned_sync = NULL + WHERE day = $1;` + _, err := tx.Exec(resetQry, day) + if err != nil { + return fmt.Errorf("error resetting sync duty validator_stats for day [%v]: %w", day, err) + } + logger.Infof("sync duty reset completed, took %v", time.Since(start)) + startEpoch, endEpoch := utils.GetFirstAndLastEpochForDay(day) + if startEpoch < utils.Config.Chain.ClConfig.AltairForkEpoch && endEpoch > utils.Config.Chain.ClConfig.AltairForkEpoch { + startEpoch = utils.Config.Chain.ClConfig.AltairForkEpoch + } else if endEpoch < utils.Config.Chain.ClConfig.AltairForkEpoch { + logger.Infof("day %v is pre-altair, skipping sync committee export", day) + return nil + } - start := time.Now() + start = time.Now() logrus.Infof("Update Sync duties for day [%v] epoch %v -> %v", day, startEpoch, endEpoch) - syncStats, err := BigtableClient.GetValidatorSyncDutiesStatistics([]uint64{}, startEpoch, endEpoch) + syncStats, err := BigtableClient.GetValidatorSyncDutiesStatistics(validators, startEpoch, endEpoch) if err != nil { return fmt.Errorf("error in GetValidatorSyncDutiesStatistics for startEpoch [%v] and endEpoch [%v]: %w", startEpoch, endEpoch, err) } @@ -1052,12 +1072,6 @@ func WriteValidatorSyncDutiesForDay(day uint64) error { syncStatsArr = append(syncStatsArr, stat) } - tx, err := WriterDb.Beginx() - if err != nil { - return err - } - defer tx.Rollback() - batchSize := 13000 // max parameters: 65535 for b := 0; b < len(syncStatsArr); b += batchSize { start := b @@ -1090,13 +1104,9 @@ func WriteValidatorSyncDutiesForDay(day uint64) error { logrus.Infof("saving sync statistics batch %v completed", b) } - if err = tx.Commit(); err != nil { - return err - } - logger.Infof("export completed, took %v", time.Since(start)) - if err = markColumnExported(day, "sync_duties_exported"); err != nil { + if err = markColumnExported(day, "sync_duties_exported", tx); err != nil { return err } @@ -1104,7 +1114,7 @@ func WriteValidatorSyncDutiesForDay(day uint64) error { return nil } -func WriteValidatorFailedAttestationsStatisticsForDay(day uint64, concurrency uint64) error { +func WriteValidatorFailedAttestationsStatisticsForDay(validators []uint64, day uint64, concurrency uint64, tx *sqlx.Tx) error { ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(time.Minute*10)) defer cancel() exportStart := time.Now() @@ -1116,56 +1126,42 @@ func WriteValidatorFailedAttestationsStatisticsForDay(day uint64, concurrency ui return err } + start := time.Now() + resetQry := ` + UPDATE validator_stats SET + missed_attestations = NULL, + orphaned_attestations = NULL + WHERE day = $1;` + _, err := tx.Exec(resetQry, day) + if err != nil { + return fmt.Errorf("error resetting attestation duty validator_stats for day [%v]: %w", day, err) + } + logger.Infof("attestation duty reset completed, took %v", time.Since(start)) + firstEpoch, lastEpoch := utils.GetFirstAndLastEpochForDay(day) - start := time.Now() + start = time.Now() logrus.Infof("exporting 'failed attestations' statistics firstEpoch: %v lastEpoch: %v", firstEpoch, lastEpoch) - // first key is the batch start index and the second is the validator id - failed := map[uint64]map[uint64]*types.ValidatorMissedAttestationsStatistic{} - mux := sync.Mutex{} - g, gCtx := errgroup.WithContext(ctx) - g.SetLimit(int(concurrency)) - epochBatchSize := uint64(2) // Fetching 2 Epochs per batch seems to be the fastest way to go - for i := firstEpoch; i < lastEpoch; i += epochBatchSize { - fromEpoch := i - toEpoch := fromEpoch + epochBatchSize - if toEpoch >= lastEpoch { - toEpoch = lastEpoch - } else { - toEpoch-- - } - g.Go(func() error { - select { - case <-gCtx.Done(): - return gCtx.Err() - default: - } - ma, err := BigtableClient.GetValidatorMissedAttestationsCount([]uint64{}, fromEpoch, toEpoch) - if err != nil { - return fmt.Errorf("error in GetValidatorMissedAttestationsCount for fromEpoch [%v] and toEpoch [%v]: %w", fromEpoch, toEpoch, err) - } - mux.Lock() - failed[fromEpoch] = ma - mux.Unlock() - return nil - }) - } + validatorMap := map[uint64]*types.ValidatorMissedAttestationsStatistic{} + batchSize := 10000 + for i := 0; i < len(validators); i += batchSize { - if err := g.Wait(); err != nil { - return err - } + upperBound := i + batchSize + if len(validators) < upperBound { + upperBound = len(validators) + } + vals := validators[i:upperBound] - validatorMap := map[uint64]*types.ValidatorMissedAttestationsStatistic{} - for _, f := range failed { + logrus.Infof("retrieving validator missed attestations stats for validators %v - %v", vals[0], vals[len(vals)-1]) - for key, val := range f { - if validatorMap[key] == nil { - validatorMap[key] = val - } else { - validatorMap[key].MissedAttestations += val.MissedAttestations - } + ma, err := BigtableClient.GetValidatorMissedAttestationsCount(vals, firstEpoch, lastEpoch) + if err != nil { + return fmt.Errorf("error in GetValidatorMissedAttestationsCount for fromEpoch [%v] and toEpoch [%v]: %w", firstEpoch, lastEpoch, err) + } + for validator, stats := range ma { + validatorMap[validator] = stats } } @@ -1177,13 +1173,13 @@ func WriteValidatorFailedAttestationsStatisticsForDay(day uint64, concurrency ui maArr = append(maArr, stat) } - g, gCtx = errgroup.WithContext(ctx) - - batchSize := 100 // max: 65535 / 4, but we are faster with smaller batches - for b := 0; b < len(maArr); b += batchSize { + g, gCtx := errgroup.WithContext(ctx) + g.SetLimit(int(concurrency)) + dbBatchSize := 100 + for b := 0; b < len(maArr); b += dbBatchSize { start := b - end := b + batchSize + end := b + dbBatchSize if len(maArr) < end { end = len(maArr) } @@ -1195,7 +1191,7 @@ func WriteValidatorFailedAttestationsStatisticsForDay(day uint64, concurrency ui default: } - err := saveFailedAttestationBatch(maArr[start:end], day) + err := saveFailedAttestationBatch(maArr[start:end], day, tx) if err != nil { return fmt.Errorf("error in saveFailedAttestationBatch for day [%v], start [%v] and end [%v]: %w", day, start, end, err) } @@ -1209,7 +1205,7 @@ func WriteValidatorFailedAttestationsStatisticsForDay(day uint64, concurrency ui } logger.Infof("export completed, took %v", time.Since(start)) - if err := markColumnExported(day, "failed_attestations_exported"); err != nil { + if err := markColumnExported(day, "failed_attestations_exported", tx); err != nil { return err } @@ -1217,7 +1213,7 @@ func WriteValidatorFailedAttestationsStatisticsForDay(day uint64, concurrency ui return nil } -func saveFailedAttestationBatch(batch []*types.ValidatorMissedAttestationsStatistic, day uint64) error { +func saveFailedAttestationBatch(batch []*types.ValidatorMissedAttestationsStatistic, day uint64, tx *sqlx.Tx) error { var failedAttestationBatchNumArgs int = 4 batchSize := len(batch) valueStrings := make([]string, 0, failedAttestationBatchNumArgs) @@ -1235,7 +1231,7 @@ func saveFailedAttestationBatch(batch []*types.ValidatorMissedAttestationsStatis %s on conflict (validatorindex, day) do update set missed_attestations = excluded.missed_attestations, orphaned_attestations = excluded.orphaned_attestations;`, strings.Join(valueStrings, ",")) - _, err := WriterDb.Exec(stmt, valueArgs...) + _, err := tx.Exec(stmt, valueArgs...) if err != nil { return fmt.Errorf("error inserting failed attestations into validator_stats for day [%v]: %w", day, err) } @@ -1243,11 +1239,11 @@ func saveFailedAttestationBatch(batch []*types.ValidatorMissedAttestationsStatis return nil } -func markColumnExported(day uint64, column string) error { +func markColumnExported(day uint64, column string, tx *sqlx.Tx) error { start := time.Now() logger.Infof("marking [%v] exported for day [%v] as completed in the status table", column, day) - _, err := WriterDb.Exec(fmt.Sprintf(` + _, err := tx.Exec(fmt.Sprintf(` INSERT INTO validator_stats_status (day, status, %[1]v) VALUES ($1, false, true) ON CONFLICT (day) @@ -1319,18 +1315,25 @@ func GetValidatorIncomeHistory(validatorIndices []uint64, lowerBoundDay uint64, // retrieve rewards for epochs not yet in stats if upperBoundDay == 65536 { - lastDay := uint64(0) + lastDay := int64(0) if len(result) > 0 { - lastDay = uint64(result[len(result)-1].Day) + lastDay = int64(result[len(result)-1].Day) } else { - lastDay, err = GetLastExportedStatisticDay() - if err != nil { + lastDayDb, err := GetLastExportedStatisticDay() + if err == nil { + lastDay = int64(lastDayDb) + } else if err == ErrNoStats { + lastDay = -1 + } else { return nil, err } } currentDay := lastDay + 1 - firstSlot := utils.GetLastBalanceInfoSlotForDay(lastDay) + 1 + firstSlot := uint64(0) + if lastDay > -1 { + firstSlot = utils.GetLastBalanceInfoSlotForDay(uint64(lastDay)) + 1 + } lastSlot := lastFinalizedEpoch * utils.Config.Chain.ClConfig.SlotsPerEpoch totalBalance := uint64(0) @@ -1355,7 +1358,12 @@ func GetValidatorIncomeHistory(validatorIndices []uint64, lowerBoundDay uint64, var lastBalance uint64 g.Go(func() error { - return GetValidatorBalanceForDay(validatorIndices, lastDay, &lastBalance) + + if lastDay < 0 { + return GetValidatorActivationBalance(validatorIndices, &lastBalance) + } else { + return GetValidatorBalanceForDay(validatorIndices, uint64(lastDay), &lastBalance) + } }) var lastDeposits uint64 @@ -1556,13 +1564,13 @@ func WriteExecutionChartSeriesForDay(day int64) error { } lastEpoch := lastSlot / int64(utils.Config.Chain.ClConfig.SlotsPerEpoch) - finalizedCount, err := CountFinalizedEpochs(firstEpoch, uint64(lastEpoch)) + latestFinalizedEpoch, err := GetLatestFinalizedEpoch() if err != nil { - return err + return fmt.Errorf("error getting latest finalized epoch from db %w", err) } - if finalizedCount < epochsPerDay { - return fmt.Errorf("delaying chart series export as not all epochs for day %v finalized. %v of %v", day, finalizedCount, epochsPerDay) + if lastEpoch > int64(latestFinalizedEpoch) { + return fmt.Errorf("delaying chart series export as not all epochs for day %v finalized. last epoch of the day [%v] last finalized epoch [%v]", day, lastEpoch, latestFinalizedEpoch) } firstBlock, err := GetBlockNumber(uint64(firstSlot)) @@ -1868,16 +1876,16 @@ func WriteGraffitiStatisticsForDay(day int64) error { } func checkIfDayIsFinalized(day uint64) error { - epochsPerDay := utils.EpochsPerDay() - firstEpoch, lastEpoch := utils.GetFirstAndLastEpochForDay(day) + _, lastEpoch := utils.GetFirstAndLastEpochForDay(day) - finalizedCount, err := CountFinalizedEpochs(firstEpoch, lastEpoch) + latestFinalizedEpoch, err := GetLatestFinalizedEpoch() if err != nil { - return err + return fmt.Errorf("error getting latest finalized epoch from db %w", err) } - if finalizedCount < epochsPerDay { - return fmt.Errorf("delaying export as not all epochs for day %v finalized. %v of %v", day, finalizedCount, epochsPerDay) + if lastEpoch > latestFinalizedEpoch { + return fmt.Errorf("delaying statistics export as not all epochs for day %v are finalized. Last epoch of the day [%v] last finalized epoch [%v]", day, lastEpoch, latestFinalizedEpoch) } + return nil } diff --git a/ens/bindings.go b/ens/bindings.go index d16e13a0ab..a02b75a666 100644 --- a/ens/bindings.go +++ b/ens/bindings.go @@ -16,6 +16,11 @@ var ensRegistrarData = &bind.MetaData{ Bin: "", } +var ensRegistarV2Data = &bind.MetaData{ + ABI: "[{\"inputs\":[{\"internalType\":\"contract BaseRegistrarImplementation\",\"name\":\"_base\",\"type\":\"address\"},{\"internalType\":\"contract IPriceOracle\",\"name\":\"_prices\",\"type\":\"address\"},{\"internalType\":\"uint256\",\"name\":\"_minCommitmentAge\",\"type\":\"uint256\"},{\"internalType\":\"uint256\",\"name\":\"_maxCommitmentAge\",\"type\":\"uint256\"},{\"internalType\":\"contract ReverseRegistrar\",\"name\":\"_reverseRegistrar\",\"type\":\"address\"},{\"internalType\":\"contract INameWrapper\",\"name\":\"_nameWrapper\",\"type\":\"address\"}],\"stateMutability\":\"nonpayable\",\"type\":\"constructor\"},{\"inputs\":[{\"internalType\":\"bytes32\",\"name\":\"commitment\",\"type\":\"bytes32\"}],\"name\":\"CommitmentTooNew\",\"type\":\"error\"},{\"inputs\":[{\"internalType\":\"bytes32\",\"name\":\"commitment\",\"type\":\"bytes32\"}],\"name\":\"CommitmentTooOld\",\"type\":\"error\"},{\"inputs\":[{\"internalType\":\"uint256\",\"name\":\"duration\",\"type\":\"uint256\"}],\"name\":\"DurationTooShort\",\"type\":\"error\"},{\"inputs\":[],\"name\":\"InsufficientValue\",\"type\":\"error\"},{\"inputs\":[],\"name\":\"MaxCommitmentAgeTooHigh\",\"type\":\"error\"},{\"inputs\":[],\"name\":\"MaxCommitmentAgeTooLow\",\"type\":\"error\"},{\"inputs\":[{\"internalType\":\"string\",\"name\":\"name\",\"type\":\"string\"}],\"name\":\"NameNotAvailable\",\"type\":\"error\"},{\"inputs\":[],\"name\":\"ResolverRequiredWhenDataSupplied\",\"type\":\"error\"},{\"inputs\":[{\"internalType\":\"bytes32\",\"name\":\"commitment\",\"type\":\"bytes32\"}],\"name\":\"UnexpiredCommitmentExists\",\"type\":\"error\"},{\"anonymous\":false,\"inputs\":[{\"indexed\":false,\"internalType\":\"string\",\"name\":\"name\",\"type\":\"string\"},{\"indexed\":true,\"internalType\":\"bytes32\",\"name\":\"label\",\"type\":\"bytes32\"},{\"indexed\":true,\"internalType\":\"address\",\"name\":\"owner\",\"type\":\"address\"},{\"indexed\":false,\"internalType\":\"uint256\",\"name\":\"baseCost\",\"type\":\"uint256\"},{\"indexed\":false,\"internalType\":\"uint256\",\"name\":\"premium\",\"type\":\"uint256\"},{\"indexed\":false,\"internalType\":\"uint256\",\"name\":\"expires\",\"type\":\"uint256\"}],\"name\":\"NameRegistered\",\"type\":\"event\"},{\"anonymous\":false,\"inputs\":[{\"indexed\":false,\"internalType\":\"string\",\"name\":\"name\",\"type\":\"string\"},{\"indexed\":true,\"internalType\":\"bytes32\",\"name\":\"label\",\"type\":\"bytes32\"},{\"indexed\":false,\"internalType\":\"uint256\",\"name\":\"cost\",\"type\":\"uint256\"},{\"indexed\":false,\"internalType\":\"uint256\",\"name\":\"expires\",\"type\":\"uint256\"}],\"name\":\"NameRenewed\",\"type\":\"event\"},{\"anonymous\":false,\"inputs\":[{\"indexed\":true,\"internalType\":\"address\",\"name\":\"previousOwner\",\"type\":\"address\"},{\"indexed\":true,\"internalType\":\"address\",\"name\":\"newOwner\",\"type\":\"address\"}],\"name\":\"OwnershipTransferred\",\"type\":\"event\"},{\"inputs\":[],\"name\":\"MIN_REGISTRATION_DURATION\",\"outputs\":[{\"internalType\":\"uint256\",\"name\":\"\",\"type\":\"uint256\"}],\"stateMutability\":\"view\",\"type\":\"function\"},{\"inputs\":[{\"internalType\":\"string\",\"name\":\"name\",\"type\":\"string\"}],\"name\":\"available\",\"outputs\":[{\"internalType\":\"bool\",\"name\":\"\",\"type\":\"bool\"}],\"stateMutability\":\"view\",\"type\":\"function\"},{\"inputs\":[{\"internalType\":\"bytes32\",\"name\":\"commitment\",\"type\":\"bytes32\"}],\"name\":\"commit\",\"outputs\":[],\"stateMutability\":\"nonpayable\",\"type\":\"function\"},{\"inputs\":[{\"internalType\":\"bytes32\",\"name\":\"\",\"type\":\"bytes32\"}],\"name\":\"commitments\",\"outputs\":[{\"internalType\":\"uint256\",\"name\":\"\",\"type\":\"uint256\"}],\"stateMutability\":\"view\",\"type\":\"function\"},{\"inputs\":[{\"internalType\":\"string\",\"name\":\"name\",\"type\":\"string\"},{\"internalType\":\"address\",\"name\":\"owner\",\"type\":\"address\"},{\"internalType\":\"uint256\",\"name\":\"duration\",\"type\":\"uint256\"},{\"internalType\":\"bytes32\",\"name\":\"secret\",\"type\":\"bytes32\"},{\"internalType\":\"address\",\"name\":\"resolver\",\"type\":\"address\"},{\"internalType\":\"bytes[]\",\"name\":\"data\",\"type\":\"bytes[]\"},{\"internalType\":\"bool\",\"name\":\"reverseRecord\",\"type\":\"bool\"},{\"internalType\":\"uint16\",\"name\":\"ownerControlledFuses\",\"type\":\"uint16\"}],\"name\":\"makeCommitment\",\"outputs\":[{\"internalType\":\"bytes32\",\"name\":\"\",\"type\":\"bytes32\"}],\"stateMutability\":\"pure\",\"type\":\"function\"},{\"inputs\":[],\"name\":\"maxCommitmentAge\",\"outputs\":[{\"internalType\":\"uint256\",\"name\":\"\",\"type\":\"uint256\"}],\"stateMutability\":\"view\",\"type\":\"function\"},{\"inputs\":[],\"name\":\"minCommitmentAge\",\"outputs\":[{\"internalType\":\"uint256\",\"name\":\"\",\"type\":\"uint256\"}],\"stateMutability\":\"view\",\"type\":\"function\"},{\"inputs\":[],\"name\":\"nameWrapper\",\"outputs\":[{\"internalType\":\"contract INameWrapper\",\"name\":\"\",\"type\":\"address\"}],\"stateMutability\":\"view\",\"type\":\"function\"},{\"inputs\":[],\"name\":\"owner\",\"outputs\":[{\"internalType\":\"address\",\"name\":\"\",\"type\":\"address\"}],\"stateMutability\":\"view\",\"type\":\"function\"},{\"inputs\":[],\"name\":\"prices\",\"outputs\":[{\"internalType\":\"contract IPriceOracle\",\"name\":\"\",\"type\":\"address\"}],\"stateMutability\":\"view\",\"type\":\"function\"},{\"inputs\":[{\"internalType\":\"address\",\"name\":\"_token\",\"type\":\"address\"},{\"internalType\":\"address\",\"name\":\"_to\",\"type\":\"address\"},{\"internalType\":\"uint256\",\"name\":\"_amount\",\"type\":\"uint256\"}],\"name\":\"recoverFunds\",\"outputs\":[],\"stateMutability\":\"nonpayable\",\"type\":\"function\"},{\"inputs\":[{\"internalType\":\"string\",\"name\":\"name\",\"type\":\"string\"},{\"internalType\":\"address\",\"name\":\"owner\",\"type\":\"address\"},{\"internalType\":\"uint256\",\"name\":\"duration\",\"type\":\"uint256\"},{\"internalType\":\"bytes32\",\"name\":\"secret\",\"type\":\"bytes32\"},{\"internalType\":\"address\",\"name\":\"resolver\",\"type\":\"address\"},{\"internalType\":\"bytes[]\",\"name\":\"data\",\"type\":\"bytes[]\"},{\"internalType\":\"bool\",\"name\":\"reverseRecord\",\"type\":\"bool\"},{\"internalType\":\"uint16\",\"name\":\"ownerControlledFuses\",\"type\":\"uint16\"}],\"name\":\"register\",\"outputs\":[],\"stateMutability\":\"payable\",\"type\":\"function\"},{\"inputs\":[{\"internalType\":\"string\",\"name\":\"name\",\"type\":\"string\"},{\"internalType\":\"uint256\",\"name\":\"duration\",\"type\":\"uint256\"}],\"name\":\"renew\",\"outputs\":[],\"stateMutability\":\"payable\",\"type\":\"function\"},{\"inputs\":[],\"name\":\"renounceOwnership\",\"outputs\":[],\"stateMutability\":\"nonpayable\",\"type\":\"function\"},{\"inputs\":[{\"internalType\":\"string\",\"name\":\"name\",\"type\":\"string\"},{\"internalType\":\"uint256\",\"name\":\"duration\",\"type\":\"uint256\"}],\"name\":\"rentPrice\",\"outputs\":[{\"components\":[{\"internalType\":\"uint256\",\"name\":\"base\",\"type\":\"uint256\"},{\"internalType\":\"uint256\",\"name\":\"premium\",\"type\":\"uint256\"}],\"internalType\":\"struct IPriceOracle.Price\",\"name\":\"price\",\"type\":\"tuple\"}],\"stateMutability\":\"view\",\"type\":\"function\"},{\"inputs\":[],\"name\":\"reverseRegistrar\",\"outputs\":[{\"internalType\":\"contract ReverseRegistrar\",\"name\":\"\",\"type\":\"address\"}],\"stateMutability\":\"view\",\"type\":\"function\"},{\"inputs\":[{\"internalType\":\"bytes4\",\"name\":\"interfaceID\",\"type\":\"bytes4\"}],\"name\":\"supportsInterface\",\"outputs\":[{\"internalType\":\"bool\",\"name\":\"\",\"type\":\"bool\"}],\"stateMutability\":\"pure\",\"type\":\"function\"},{\"inputs\":[{\"internalType\":\"address\",\"name\":\"newOwner\",\"type\":\"address\"}],\"name\":\"transferOwnership\",\"outputs\":[],\"stateMutability\":\"nonpayable\",\"type\":\"function\"},{\"inputs\":[{\"internalType\":\"string\",\"name\":\"name\",\"type\":\"string\"}],\"name\":\"valid\",\"outputs\":[{\"internalType\":\"bool\",\"name\":\"\",\"type\":\"bool\"}],\"stateMutability\":\"pure\",\"type\":\"function\"},{\"inputs\":[],\"name\":\"withdraw\",\"outputs\":[],\"stateMutability\":\"nonpayable\",\"type\":\"function\"}]", + Bin: "", +} + var ensResolverControllerData = &bind.MetaData{ ABI: "[{\"inputs\":[{\"internalType\":\"contract ENS\",\"name\":\"_old\",\"type\":\"address\"}],\"payable\":false,\"stateMutability\":\"nonpayable\",\"type\":\"constructor\"},{\"anonymous\":false,\"inputs\":[{\"indexed\":true,\"internalType\":\"address\",\"name\":\"owner\",\"type\":\"address\"},{\"indexed\":true,\"internalType\":\"address\",\"name\":\"operator\",\"type\":\"address\"},{\"indexed\":false,\"internalType\":\"bool\",\"name\":\"approved\",\"type\":\"bool\"}],\"name\":\"ApprovalForAll\",\"type\":\"event\"},{\"anonymous\":false,\"inputs\":[{\"indexed\":true,\"internalType\":\"bytes32\",\"name\":\"node\",\"type\":\"bytes32\"},{\"indexed\":true,\"internalType\":\"bytes32\",\"name\":\"label\",\"type\":\"bytes32\"},{\"indexed\":false,\"internalType\":\"address\",\"name\":\"owner\",\"type\":\"address\"}],\"name\":\"NewOwner\",\"type\":\"event\"},{\"anonymous\":false,\"inputs\":[{\"indexed\":true,\"internalType\":\"bytes32\",\"name\":\"node\",\"type\":\"bytes32\"},{\"indexed\":false,\"internalType\":\"address\",\"name\":\"resolver\",\"type\":\"address\"}],\"name\":\"NewResolver\",\"type\":\"event\"},{\"anonymous\":false,\"inputs\":[{\"indexed\":true,\"internalType\":\"bytes32\",\"name\":\"node\",\"type\":\"bytes32\"},{\"indexed\":false,\"internalType\":\"uint64\",\"name\":\"ttl\",\"type\":\"uint64\"}],\"name\":\"NewTTL\",\"type\":\"event\"},{\"anonymous\":false,\"inputs\":[{\"indexed\":true,\"internalType\":\"bytes32\",\"name\":\"node\",\"type\":\"bytes32\"},{\"indexed\":false,\"internalType\":\"address\",\"name\":\"owner\",\"type\":\"address\"}],\"name\":\"Transfer\",\"type\":\"event\"},{\"constant\":true,\"inputs\":[{\"internalType\":\"address\",\"name\":\"owner\",\"type\":\"address\"},{\"internalType\":\"address\",\"name\":\"operator\",\"type\":\"address\"}],\"name\":\"isApprovedForAll\",\"outputs\":[{\"internalType\":\"bool\",\"name\":\"\",\"type\":\"bool\"}],\"payable\":false,\"stateMutability\":\"view\",\"type\":\"function\"},{\"constant\":true,\"inputs\":[],\"name\":\"old\",\"outputs\":[{\"internalType\":\"contract ENS\",\"name\":\"\",\"type\":\"address\"}],\"payable\":false,\"stateMutability\":\"view\",\"type\":\"function\"},{\"constant\":true,\"inputs\":[{\"internalType\":\"bytes32\",\"name\":\"node\",\"type\":\"bytes32\"}],\"name\":\"owner\",\"outputs\":[{\"internalType\":\"address\",\"name\":\"\",\"type\":\"address\"}],\"payable\":false,\"stateMutability\":\"view\",\"type\":\"function\"},{\"constant\":true,\"inputs\":[{\"internalType\":\"bytes32\",\"name\":\"node\",\"type\":\"bytes32\"}],\"name\":\"recordExists\",\"outputs\":[{\"internalType\":\"bool\",\"name\":\"\",\"type\":\"bool\"}],\"payable\":false,\"stateMutability\":\"view\",\"type\":\"function\"},{\"constant\":true,\"inputs\":[{\"internalType\":\"bytes32\",\"name\":\"node\",\"type\":\"bytes32\"}],\"name\":\"resolver\",\"outputs\":[{\"internalType\":\"address\",\"name\":\"\",\"type\":\"address\"}],\"payable\":false,\"stateMutability\":\"view\",\"type\":\"function\"},{\"constant\":false,\"inputs\":[{\"internalType\":\"address\",\"name\":\"operator\",\"type\":\"address\"},{\"internalType\":\"bool\",\"name\":\"approved\",\"type\":\"bool\"}],\"name\":\"setApprovalForAll\",\"outputs\":[],\"payable\":false,\"stateMutability\":\"nonpayable\",\"type\":\"function\"},{\"constant\":false,\"inputs\":[{\"internalType\":\"bytes32\",\"name\":\"node\",\"type\":\"bytes32\"},{\"internalType\":\"address\",\"name\":\"owner\",\"type\":\"address\"}],\"name\":\"setOwner\",\"outputs\":[],\"payable\":false,\"stateMutability\":\"nonpayable\",\"type\":\"function\"},{\"constant\":false,\"inputs\":[{\"internalType\":\"bytes32\",\"name\":\"node\",\"type\":\"bytes32\"},{\"internalType\":\"address\",\"name\":\"owner\",\"type\":\"address\"},{\"internalType\":\"address\",\"name\":\"resolver\",\"type\":\"address\"},{\"internalType\":\"uint64\",\"name\":\"ttl\",\"type\":\"uint64\"}],\"name\":\"setRecord\",\"outputs\":[],\"payable\":false,\"stateMutability\":\"nonpayable\",\"type\":\"function\"},{\"constant\":false,\"inputs\":[{\"internalType\":\"bytes32\",\"name\":\"node\",\"type\":\"bytes32\"},{\"internalType\":\"address\",\"name\":\"resolver\",\"type\":\"address\"}],\"name\":\"setResolver\",\"outputs\":[],\"payable\":false,\"stateMutability\":\"nonpayable\",\"type\":\"function\"},{\"constant\":false,\"inputs\":[{\"internalType\":\"bytes32\",\"name\":\"node\",\"type\":\"bytes32\"},{\"internalType\":\"bytes32\",\"name\":\"label\",\"type\":\"bytes32\"},{\"internalType\":\"address\",\"name\":\"owner\",\"type\":\"address\"}],\"name\":\"setSubnodeOwner\",\"outputs\":[{\"internalType\":\"bytes32\",\"name\":\"\",\"type\":\"bytes32\"}],\"payable\":false,\"stateMutability\":\"nonpayable\",\"type\":\"function\"},{\"constant\":false,\"inputs\":[{\"internalType\":\"bytes32\",\"name\":\"node\",\"type\":\"bytes32\"},{\"internalType\":\"bytes32\",\"name\":\"label\",\"type\":\"bytes32\"},{\"internalType\":\"address\",\"name\":\"owner\",\"type\":\"address\"},{\"internalType\":\"address\",\"name\":\"resolver\",\"type\":\"address\"},{\"internalType\":\"uint64\",\"name\":\"ttl\",\"type\":\"uint64\"}],\"name\":\"setSubnodeRecord\",\"outputs\":[],\"payable\":false,\"stateMutability\":\"nonpayable\",\"type\":\"function\"},{\"constant\":false,\"inputs\":[{\"internalType\":\"bytes32\",\"name\":\"node\",\"type\":\"bytes32\"},{\"internalType\":\"uint64\",\"name\":\"ttl\",\"type\":\"uint64\"}],\"name\":\"setTTL\",\"outputs\":[],\"payable\":false,\"stateMutability\":\"nonpayable\",\"type\":\"function\"},{\"constant\":true,\"inputs\":[{\"internalType\":\"bytes32\",\"name\":\"node\",\"type\":\"bytes32\"}],\"name\":\"ttl\",\"outputs\":[{\"internalType\":\"uint64\",\"name\":\"\",\"type\":\"uint64\"}],\"payable\":false,\"stateMutability\":\"view\",\"type\":\"function\"}]", Bin: "", @@ -36,6 +41,17 @@ type NameRegistered struct { Raw types.Log // Blockchain specific contextual infos } +// NameRegisteredV2 represents an NameRegistered event raised by the Ens Registar contract. +type NameRegisteredV2 struct { + Name string + Label [32]byte + Owner common.Address + BaseCost *big.Int + Premium *big.Int + Expires *big.Int + Raw types.Log // Blockchain specific contextual infos +} + // NameRenewed represents an NameRenewed event raised by the Ens Registar contract. type NameRenewed struct { Name string @@ -78,6 +94,7 @@ type NewResolver struct { // EnsFilterer is a log filtering Go binding around an Ethereum contract events. type EnsRegistrarFilterer struct { contract *bind.BoundContract // Generic contract wrapper for the low level calls + contractV2 *bind.BoundContract // Second version of registrar contract resolverControllerContract *bind.BoundContract // contract wrapper for resolver controller contract resolverContract *bind.BoundContract // contract wrapper for resolver contract } @@ -88,6 +105,10 @@ func NewEnsRegistrarFilterer(address common.Address, filterer bind.ContractFilte if err != nil { return nil, err } + contractV2, err := bindEnsRegistarV2Controller(address, nil, nil, filterer) + if err != nil { + return nil, err + } resolverControllerContract, err := bindEnsResolverController(address, nil, nil, filterer) if err != nil { return nil, err @@ -98,6 +119,7 @@ func NewEnsRegistrarFilterer(address common.Address, filterer bind.ContractFilte } return &EnsRegistrarFilterer{ contract: contract, + contractV2: contractV2, resolverControllerContract: resolverControllerContract, resolverContract: resolverContract}, nil } @@ -111,6 +133,15 @@ func bindEnsRegistarController(address common.Address, caller bind.ContractCalle return bind.NewBoundContract(address, parsed, caller, transactor, filterer), nil } +// bindEnsRegistarController binds a generic wrapper to an already deployed contract. +func bindEnsRegistarV2Controller(address common.Address, caller bind.ContractCaller, transactor bind.ContractTransactor, filterer bind.ContractFilterer) (*bind.BoundContract, error) { + parsed, err := abi.JSON(strings.NewReader(ensRegistarV2Data.ABI)) + if err != nil { + return nil, err + } + return bind.NewBoundContract(address, parsed, caller, transactor, filterer), nil +} + // bindEnsResolverController binds a generic wrapper to an already deployed contract. func bindEnsResolverController(address common.Address, caller bind.ContractCaller, transactor bind.ContractTransactor, filterer bind.ContractFilterer) (*bind.BoundContract, error) { parsed, err := abi.JSON(strings.NewReader(ensResolverControllerData.ABI)) @@ -139,6 +170,16 @@ func (_EnsRegistrar *EnsRegistrarFilterer) ParseNameRegistered(log types.Log) (* return event, nil } +// Solidity: event NameRegistered(string name, index_topic_1 bytes32 label, index_topic_2 address owner, uint256 baseCost, uint256 premium, uint256 expires); +func (_EnsRegistrar *EnsRegistrarFilterer) ParseNameRegisteredV2(log types.Log) (*NameRegisteredV2, error) { + event := new(NameRegisteredV2) + if err := _EnsRegistrar.contractV2.UnpackLog(event, "NameRegistered", log); err != nil { + return nil, err + } + event.Raw = log + return event, nil +} + // Solidity: event NewResolver (index_topic_1 bytes32 node, address resolver) func (_EnsRegistrar *EnsRegistrarFilterer) ParseNewResolver(log types.Log) (*NewResolver, error) { event := new(NewResolver) diff --git a/ens/ens.go b/ens/ens.go index aa054570ee..22c0f917ac 100644 --- a/ens/ens.go +++ b/ens/ens.go @@ -6,6 +6,9 @@ var NewResolverTopic []byte = []byte{0x33, 0x57, 0x21, 0xb0, 0x18, 0x66, 0xdc, 0 // ca6abbe9d7f11422cb6ca7629fbf6fe9efb1c621f71ce8f02b9f2a230097404f var NameRegisteredTopic []byte = []byte{0xca, 0x6a, 0xbb, 0xe9, 0xd7, 0xf1, 0x14, 0x22, 0xcb, 0x6c, 0xa7, 0x62, 0x9f, 0xbf, 0x6f, 0xe9, 0xef, 0xb1, 0xc6, 0x21, 0xf7, 0x1c, 0xe8, 0xf0, 0x2b, 0x9f, 0x2a, 0x23, 0x00, 0x97, 0x40, 0x4f} +// 69e37f151eb98a09618ddaa80c8cfaf1ce5996867c489f45b555b412271ebf27 +var NameRegisteredV2Topic []byte = []byte{0x69, 0xe3, 0x7f, 0x15, 0x1e, 0xb9, 0x8a, 0x09, 0x61, 0x8d, 0xda, 0xa8, 0x0c, 0x8c, 0xfa, 0xf1, 0xce, 0x59, 0x96, 0x86, 0x7c, 0x48, 0x9f, 0x45, 0xb5, 0x55, 0xb4, 0x12, 0x27, 0x1e, 0xbf, 0x27} + // 3da24c024582931cfaf8267d8ed24d13a82a8068d5bd337d30ec45cea4e506ae var NameRenewedTopic []byte = []byte{0x3d, 0xa2, 0x4c, 0x02, 0x45, 0x82, 0x93, 0x1c, 0xfa, 0xf8, 0x26, 0x7d, 0x8e, 0xd2, 0x4d, 0x13, 0xa8, 0x2a, 0x80, 0x68, 0xd5, 0xbd, 0x33, 0x7d, 0x30, 0xec, 0x45, 0xce, 0xa4, 0xe5, 0x06, 0xae} diff --git a/erc20/erc20.go b/erc20/erc20.go index ead654a2bf..c6660327d0 100644 --- a/erc20/erc20.go +++ b/erc20/erc20.go @@ -4,8 +4,8 @@ import ( "encoding/json" "eth2-exporter/utils" "fmt" - "io/ioutil" "math/big" + "os" "strings" "github.com/ethereum/go-ethereum/accounts/abi" @@ -22,7 +22,7 @@ var tokenMap = make(map[string]*ERC20TokenDetail) var logger = logrus.StandardLogger().WithField("module", "erc20") func InitTokenList(path string) { - body, err := ioutil.ReadFile(path) + body, err := os.ReadFile(path) if err != nil { utils.LogFatal(err, "unable to retrieve erc20 token list", 0) } diff --git a/eth1data/eth1data.go b/eth1data/eth1data.go index 94c119a45a..f9ed2e5c10 100644 --- a/eth1data/eth1data.go +++ b/eth1data/eth1data.go @@ -7,6 +7,7 @@ import ( "eth2-exporter/cache" "eth2-exporter/db" "eth2-exporter/rpc" + "eth2-exporter/services" "eth2-exporter/types" "eth2-exporter/utils" "fmt" @@ -17,6 +18,7 @@ import ( "github.com/ethereum/go-ethereum/accounts/abi" "github.com/ethereum/go-ethereum/accounts/abi/bind" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core" geth_types "github.com/ethereum/go-ethereum/core/types" "github.com/sirupsen/logrus" ) @@ -35,18 +37,14 @@ func GetEth1Transaction(hash common.Hash) (*types.Eth1TxData, error) { data := wanted.(*types.Eth1TxData) if data.BlockNumber != 0 { - err := db.ReaderDb.Get(&data.Epoch, - `select epochs.finalized, epochs.globalparticipationrate from blocks left join epochs on blocks.epoch = epochs.epoch where blocks.exec_block_number = $1 and blocks.status='1';`, - data.BlockNumber) - if err != nil { + if err := db.GetBlockStatus(data.BlockNumber, services.LatestFinalizedEpoch(), &data.Epoch); err != nil { logger.Warningf("failed to get finalization stats for block %v", data.BlockNumber) data.Epoch.Finalized = false data.Epoch.Participation = -1 } - } - return data, nil - } + return data, nil + } */ tx, pending, err := rpc.CurrentErigonClient.GetNativeClient().TransactionByHash(ctx, hash) @@ -91,54 +89,37 @@ func GetEth1Transaction(hash common.Hash) (*types.Eth1TxData, error) { txPageData.BlockNumber = header.Number.Int64() txPageData.Timestamp = time.Unix(int64(header.Time), 0) - // msg, err := tx.AsMessage(geth_types.NewLondonSigner(tx.ChainId()), header.BaseFee) - // if err != nil { - // return nil, fmt.Errorf("error converting tx %v to message: %v", hash, err) - // } - - sender, err := geth_types.Sender(geth_types.NewCancunSigner(tx.ChainId()), tx) + msg, err := core.TransactionToMessage(tx, geth_types.NewLondonSigner(tx.ChainId()), header.BaseFee) if err != nil { return nil, fmt.Errorf("error getting sender of tx %v: %w", hash, err) } - - txPageData.From = common.BytesToAddress(sender.Bytes()) - txPageData.Nonce = tx.Nonce() + txPageData.From = msg.From + txPageData.Nonce = msg.Nonce txPageData.Type = receipt.Type txPageData.TypeFormatted = utils.FormatTransactionType(receipt.Type) txPageData.TxnPosition = receipt.TransactionIndex - txPageData.Gas.MaxPriorityFee = tx.GasTipCap().Bytes() - txPageData.Gas.MaxFee = tx.GasFeeCap().Bytes() + txPageData.Gas.MaxPriorityFee = msg.GasTipCap.Bytes() + txPageData.Gas.MaxFee = msg.GasFeeCap.Bytes() if header.BaseFee != nil { txPageData.Gas.BlockBaseFee = header.BaseFee.Bytes() } txPageData.Gas.Used = receipt.GasUsed - txPageData.Gas.Limit = tx.Gas() - txPageData.Gas.UsedPerc = float64(receipt.GasUsed) / float64(tx.Gas()) + txPageData.Gas.Limit = msg.GasLimit + txPageData.Gas.UsedPerc = float64(receipt.GasUsed) / float64(msg.GasLimit) if receipt.Type >= 2 { tmp := new(big.Int) tmp.Add(tmp, header.BaseFee) - if t := *new(big.Int).Sub(tx.GasFeeCap(), tmp); t.Cmp(tx.GasTipCap()) == -1 { + if t := *new(big.Int).Sub(msg.GasFeeCap, tmp); t.Cmp(msg.GasTipCap) == -1 { tmp.Add(tmp, &t) } else { - tmp.Add(tmp, tx.GasTipCap()) + tmp.Add(tmp, msg.GasTipCap) } txPageData.Gas.EffectiveFee = tmp.Bytes() txPageData.Gas.TxFee = tmp.Mul(tmp, big.NewInt(int64(receipt.GasUsed))).Bytes() } else { - txPageData.Gas.EffectiveFee = tx.GasFeeCap().Bytes() - txPageData.Gas.TxFee = tx.GasFeeCap().Mul(tx.GasFeeCap(), big.NewInt(int64(receipt.GasUsed))).Bytes() - } - - if receipt.Type == 3 { - txPageData.Gas.BlobGasPrice = receipt.BlobGasPrice.Bytes() - txPageData.Gas.BlobGasUsed = receipt.BlobGasUsed - txPageData.Gas.BlobFee = new(big.Int).Mul(receipt.BlobGasPrice, big.NewInt(int64(txPageData.Gas.BlobGasUsed))).Bytes() - - txPageData.BlobHashes = make([][]byte, len(tx.BlobHashes())) - for i, h := range tx.BlobHashes() { - txPageData.BlobHashes[i] = h.Bytes() - } + txPageData.Gas.EffectiveFee = msg.GasFeeCap.Bytes() + txPageData.Gas.TxFee = msg.GasFeeCap.Mul(msg.GasFeeCap, big.NewInt(int64(receipt.GasUsed))).Bytes() } if receipt.Status != 1 { @@ -156,17 +137,17 @@ func GetEth1Transaction(hash common.Hash) (*types.Eth1TxData, error) { if err != nil { return nil, fmt.Errorf("error loading token transfers from tx %v: %v", hash, err) } - txPageData.InternalTxns, err = db.BigtableClient.GetInternalTransfersForTransaction(tx.Hash().Bytes(), txPageData.From.Bytes()) + txPageData.InternalTxns, err = db.BigtableClient.GetInternalTransfersForTransaction(tx.Hash().Bytes(), msg.From.Bytes()) if err != nil { return nil, fmt.Errorf("error loading internal transfers from tx %v: %v", hash, err) } } - txPageData.FromName, err = db.BigtableClient.GetAddressName(txPageData.From.Bytes()) + txPageData.FromName, err = db.BigtableClient.GetAddressName(msg.From.Bytes()) if err != nil { return nil, fmt.Errorf("error retrieveing from name for tx %v: %v", hash, err) } - if tx.To() != nil { - txPageData.ToName, err = db.BigtableClient.GetAddressName(tx.To().Bytes()) + if msg.To != nil { + txPageData.ToName, err = db.BigtableClient.GetAddressName(msg.To.Bytes()) if err != nil { return nil, fmt.Errorf("error retrieveing to name for tx %v: %v", hash, err) } @@ -247,10 +228,7 @@ func GetEth1Transaction(hash common.Hash) (*types.Eth1TxData, error) { } if txPageData.BlockNumber != 0 { - err := db.ReaderDb.Get(&txPageData.Epoch, - `select epochs.finalized, epochs.globalparticipationrate from blocks left join epochs on blocks.epoch = epochs.epoch where blocks.exec_block_number = $1 and blocks.status='1';`, - &txPageData.BlockNumber) - if err != nil { + if err := db.GetBlockStatus(txPageData.BlockNumber, services.LatestFinalizedEpoch(), &txPageData.Epoch); err != nil { logger.Warningf("failed to get finalization stats for block %v: %v", txPageData.BlockNumber, err) txPageData.Epoch.Finalized = false txPageData.Epoch.Participation = -1 diff --git a/exporter/appsubscription_oracle.go b/exporter/appsubscription_oracle.go index afcfa108f8..bd640bb252 100644 --- a/exporter/appsubscription_oracle.go +++ b/exporter/appsubscription_oracle.go @@ -8,8 +8,8 @@ import ( "eth2-exporter/types" "eth2-exporter/utils" "fmt" - "io/ioutil" "log" + "os" "strings" "time" @@ -98,7 +98,7 @@ func VerifyReceipt(googleClient *playstore.Client, receipt *types.PremiumData) ( } func initGoogle() *playstore.Client { - jsonKey, err := ioutil.ReadFile(utils.Config.Frontend.AppSubsGoogleJSONPath) + jsonKey, err := os.ReadFile(utils.Config.Frontend.AppSubsGoogleJSONPath) if err != nil { log.Fatal(err) } diff --git a/exporter/eth1.go b/exporter/eth1.go index 1f389d482c..b2531908d1 100644 --- a/exporter/eth1.go +++ b/exporter/eth1.go @@ -88,15 +88,19 @@ func eth1DepositsExporter() { fromBlock = lastFetchedBlock + 1 } // if we are not synced to the head yet fetch missing blocks in batches of size 1000 - if toBlock-fromBlock > eth1MaxFetch { - toBlock = fromBlock + 1000 + if toBlock > fromBlock+eth1MaxFetch { + toBlock = fromBlock + eth1MaxFetch } if toBlock > blockHeight { toBlock = blockHeight } // if we are synced to the head look at the last 100 blocks - if (toBlock-fromBlock < eth1LookBack) && (toBlock > eth1LookBack) { - fromBlock = toBlock - eth1LookBack + if toBlock < fromBlock+eth1LookBack { + if toBlock > eth1LookBack { + fromBlock = toBlock - eth1LookBack + } else { + fromBlock = 1 + } } depositsToSave, err := fetchEth1Deposits(fromBlock, toBlock) @@ -135,13 +139,15 @@ func eth1DepositsExporter() { // make sure we are progressing even if there are no deposits in the last batch lastFetchedBlock = toBlock - logger.WithFields(logrus.Fields{ - "duration": time.Since(t0), - "blockHeight": blockHeight, - "fromBlock": fromBlock, - "toBlock": toBlock, - "depositsSaved": len(depositsToSave), - }).Info("exported eth1-deposits") + if len(depositsToSave) > 0 { + logger.WithFields(logrus.Fields{ + "duration": time.Since(t0), + "blockHeight": blockHeight, + "fromBlock": fromBlock, + "toBlock": toBlock, + "depositsSaved": len(depositsToSave), + }).Info("exported eth1-deposits") + } // progress faster if we are not synced to head yet if blockHeight != toBlock { diff --git a/exporter/ethstore.go b/exporter/ethstore.go index a4c65435e4..7fbe55c521 100644 --- a/exporter/ethstore.go +++ b/exporter/ethstore.go @@ -27,7 +27,7 @@ type EthStoreExporter struct { } // start exporting of eth.store into db -func StartEthStoreExporter(bnAddress string, enAddress string, updateInterval, errorInterval, sleepInterval time.Duration) { +func StartEthStoreExporter(bnAddress string, enAddress string, updateInterval, errorInterval, sleepInterval time.Duration, startDayReexport, endDayReexport int64) { logger.Info("starting eth.store exporter") ese := &EthStoreExporter{ DB: db.WriterDb, @@ -48,22 +48,71 @@ func StartEthStoreExporter(bnAddress string, enAddress string, updateInterval, e ese.Sleep = time.Minute } - ese.Run() + // Reexport days if specified + if startDayReexport != -1 && endDayReexport != -1 { + for day := startDayReexport; day <= endDayReexport; day++ { + err := ese.reexportDay(strconv.FormatInt(day, 10)) + if err != nil { + utils.LogError(err, fmt.Sprintf("error reexporting eth.store day %d in database", day), 0) + return + } + } + return + } + ese.Run() } -func (ese *EthStoreExporter) ExportDay(day string) error { - ethStoreDay, validators, err := ese.getStoreDay(day) +func (ese *EthStoreExporter) reexportDay(day string) error { + tx, err := ese.DB.Beginx() if err != nil { return err } + defer tx.Rollback() + ese.prepareClearDayTx(tx, day) + if err != nil { + return err + } + + ese.prepareExportDayTx(tx, day) + if err != nil { + return err + } + + return tx.Commit() +} + +func (ese *EthStoreExporter) exportDay(day string) error { tx, err := ese.DB.Beginx() if err != nil { return err } defer tx.Rollback() + err = ese.prepareExportDayTx(tx, day) + if err != nil { + return err + } + + return tx.Commit() +} + +func (ese *EthStoreExporter) prepareClearDayTx(tx *sqlx.Tx, day string) error { + dayInt, err := strconv.ParseInt(day, 10, 64) + if err != nil { + return err + } + _, err = tx.Exec(`DELETE FROM eth_store_stats WHERE day = $1`, dayInt) + return err +} + +func (ese *EthStoreExporter) prepareExportDayTx(tx *sqlx.Tx, day string) error { + ethStoreDay, validators, err := ese.getStoreDay(day) + if err != nil { + return err + } + numArgs := 10 batchSize := 65535 / numArgs // max 65535 params per batch, since postgres uses int16 for binding input params valueArgs := make([]interface{}, 0, batchSize*numArgs) @@ -155,10 +204,8 @@ func (ese *EthStoreExporter) ExportDay(day string) error { total_rewards_wei = excluded.total_rewards_wei, apr = excluded.apr`, ethStoreDay.Day) - if err != nil { - return err - } - return tx.Commit() + + return err } func (ese *EthStoreExporter) getStoreDay(day string) (*ethstore.Day, map[uint64]*ethstore.Day, error) { @@ -172,10 +219,15 @@ func (ese *EthStoreExporter) Run() { DBCHECK: for { // get latest eth.store day - var latestFinalizedEpoch uint64 - err := db.WriterDb.Get(&latestFinalizedEpoch, "SELECT COALESCE(MAX(epoch), 0) FROM epochs where finalized is true") + latestFinalizedEpoch, err := db.GetLatestFinalizedEpoch() if err != nil { - logger.WithError(err).Error("error retrieving latest finalized epoch from db") + utils.LogError(err, "error retrieving latest finalized epoch from db", 0) + time.Sleep(ese.ErrorInterval) + continue + } + + if latestFinalizedEpoch == 0 { + utils.LogError(err, "error retrieved 0 as latest finalized epoch from the db", 0) time.Sleep(ese.ErrorInterval) continue } @@ -188,7 +240,7 @@ DBCHECK: SELECT COUNT(*) FROM eth_store_stats WHERE validator = -1`) if err != nil { - logger.WithError(err).Error("error retrieving eth.store days count from db") + utils.LogError(err, "error retrieving eth.store days count from db", 0) time.Sleep(ese.ErrorInterval) continue } @@ -210,7 +262,7 @@ DBCHECK: SELECT day FROM eth_store_stats WHERE validator = -1`) if err != nil { - logger.WithError(err).Error("error retrieving eth.store days from db") + utils.LogError(err, "error retrieving eth.store days from db", 0) time.Sleep(ese.ErrorInterval) continue } @@ -230,9 +282,9 @@ DBCHECK: }) // export missing days for _, dayToExport := range daysToExportArray { - err = ese.ExportDay(strconv.FormatUint(dayToExport, 10)) + err = ese.exportDay(strconv.FormatUint(dayToExport, 10)) if err != nil { - logger.WithError(err).Errorf("error exporting eth.store day %d into database", dayToExport) + utils.LogError(err, fmt.Sprintf("error exporting eth.store day %d into database", dayToExport), 0) time.Sleep(ese.ErrorInterval) continue DBCHECK } diff --git a/exporter/exporter.go b/exporter/exporter.go index c8eb48bc37..6031ed1417 100644 --- a/exporter/exporter.go +++ b/exporter/exporter.go @@ -55,7 +55,6 @@ func Start(client rpc.Client) error { head, err := client.GetChainHead() if err == nil { logger.Infof("Beacon node is available with head slot: %v", head.HeadSlot) - // if we are still waiting for genesis, export epoch 0 with genesis data // epoch 0 will only contain genesis data at this point if head.HeadSlot == 0 { @@ -110,6 +109,7 @@ func Start(client rpc.Client) error { if utils.Config.Indexer.IndexMissingEpochsOnStartup { // Add any missing epoch to the export set (might happen if the indexer was stopped for a long period of time) + logger.Infof("checking for missing epochs") epochs, err := db.GetAllEpochs() if err != nil { utils.LogFatal(err, "getting all epochs from db error", 0) @@ -258,16 +258,34 @@ func Start(client rpc.Client) error { } blocksMap[block.Slot][fmt.Sprintf("%x", block.BlockRoot)] = block - err := db.BigtableClient.SaveAttestations(blocksMap) + syncDuties := make(map[types.Slot]map[types.ValidatorIndex]bool) + syncDuties[types.Slot(block.Slot)] = make(map[types.ValidatorIndex]bool) + + for validator, duty := range block.SyncDuties { + syncDuties[types.Slot(block.Slot)][types.ValidatorIndex(validator)] = duty + } + + attDuties := make(map[types.Slot]map[types.ValidatorIndex][]types.Slot) + for validator, attestedSlot := range block.AttestationDuties { + if attDuties[types.Slot(attestedSlot)] == nil { + attDuties[types.Slot(attestedSlot)] = make(map[types.ValidatorIndex][]types.Slot) + } + if attDuties[types.Slot(attestedSlot)][types.ValidatorIndex(validator)] == nil { + attDuties[types.Slot(attestedSlot)][types.ValidatorIndex(validator)] = []types.Slot{} + } + attDuties[types.Slot(attestedSlot)][types.ValidatorIndex(validator)] = append(attDuties[types.Slot(attestedSlot)][types.ValidatorIndex(validator)], types.Slot(block.Slot)) + } + + err := db.BigtableClient.SaveAttestationDuties(attDuties) if err != nil { logrus.Errorf("error exporting attestations to bigtable for block %v: %v", block.Slot, err) } - err = db.BigtableClient.SaveSyncComitteeDuties(blocksMap) + err = db.BigtableClient.SaveSyncComitteeDuties(syncDuties) if err != nil { logrus.Errorf("error exporting sync committee duties to bigtable for block %v: %v", block.Slot, err) } - err = db.SaveBlock(block) + err = db.SaveBlock(block, false) if err != nil { logger.Errorf("error saving block: %v", err) } @@ -426,6 +444,14 @@ func doFullCheck(client rpc.Client, lookback uint64) { } } logger.Printf("finished export for epoch %v", epoch) + + if len(keys) > 10 && epoch%10 == 0 && epoch >= 10 { // this ensures that epoch finalization is properly updated during long running exports + logger.Infof("updating status of epochs %v-%v", startEpoch, head.HeadEpoch) + err = updateEpochStatus(client, epoch-10, epoch) + if err != nil { + logger.Errorf("error updating epoch stratus: %v", err) + } + } } logger.Infof("marking orphaned blocks of epochs %v-%v", startEpoch, head.HeadEpoch) @@ -455,11 +481,6 @@ func doFullCheck(client rpc.Client, lookback uint64) { if err != nil { logger.Errorf("error updating epoch stratus: %v", err) } - // set all finalized epochs to finalized - err = db.UpdateEpochFinalization(head.FinalizedEpoch) - if err != nil { - logger.Errorf("error updating finalization of epochs: %v", err) - } logger.Infof("exporting validation queue") err = exportValidatorQueue(client) @@ -535,7 +556,7 @@ func ExportEpoch(epoch uint64, client rpc.Client) error { // export epoch data to bigtable g := new(errgroup.Group) - g.SetLimit(7) + g.SetLimit(5) g.Go(func() error { err = db.BigtableClient.SaveValidatorBalances(epoch, data.Validators) if err != nil { @@ -543,13 +564,6 @@ func ExportEpoch(epoch uint64, client rpc.Client) error { } return nil }) - g.Go(func() error { - err = db.BigtableClient.SaveAttestationAssignments(epoch, data.ValidatorAssignmentes.AttestorAssignments) - if err != nil { - return fmt.Errorf("error exporting attestation assignments to bigtable: %v", err) - } - return nil - }) g.Go(func() error { err = db.BigtableClient.SaveProposalAssignments(epoch, data.ValidatorAssignmentes.ProposerAssignments) if err != nil { @@ -558,7 +572,7 @@ func ExportEpoch(epoch uint64, client rpc.Client) error { return nil }) g.Go(func() error { - err = db.BigtableClient.SaveAttestations(data.Blocks) + err = db.BigtableClient.SaveAttestationDuties(data.AttestationDuties) if err != nil { return fmt.Errorf("error exporting attestations to bigtable: %v", err) } @@ -572,7 +586,7 @@ func ExportEpoch(epoch uint64, client rpc.Client) error { return nil }) g.Go(func() error { - err = db.BigtableClient.SaveSyncComitteeDuties(data.Blocks) + err = db.BigtableClient.SaveSyncComitteeDuties(data.SyncDuties) if err != nil { return fmt.Errorf("error exporting sync committee duties to bigtable: %v", err) } diff --git a/exporter/rocketpool.go b/exporter/rocketpool.go index 05eabb370f..82e917b1a1 100644 --- a/exporter/rocketpool.go +++ b/exporter/rocketpool.go @@ -3,7 +3,7 @@ package exporter import ( "encoding/json" "fmt" - "io/ioutil" + "io" "math" "math/big" "net/http" @@ -21,6 +21,7 @@ import ( _ "github.com/jackc/pgx/v4/stdlib" "github.com/jmoiron/sqlx" "github.com/klauspost/compress/zstd" + "github.com/lib/pq" rpDAO "github.com/rocket-pool/rocketpool-go/dao" rpDAOTrustedNode "github.com/rocket-pool/rocketpool-go/dao/trustednode" "github.com/rocket-pool/rocketpool-go/minipool" @@ -1117,6 +1118,7 @@ func (rp *RocketpoolExporter) SaveDAOMembers() error { valueStrings := make([]string, 0, batchSize) valueArgs := make([]interface{}, 0, batchSize*nArgs) + addresses := make([][]byte, 0, batchSize) for i, d := range data[start:end] { for j := 0; j < nArgs; j++ { valueStringsArgs[j] = i*nArgs + j + 1 @@ -1130,12 +1132,39 @@ func (rp *RocketpoolExporter) SaveDAOMembers() error { valueArgs = append(valueArgs, d.LastProposalTime) valueArgs = append(valueArgs, d.RPLBondAmount.String()) valueArgs = append(valueArgs, d.UnbondedValidatorCount) + addresses = append(addresses, d.Address) } - stmt := fmt.Sprintf(`insert into rocketpool_dao_members (rocketpool_storage_address, address, id, url, joined_time, last_proposal_time, rpl_bond_amount, unbonded_validator_count) values %s on conflict (rocketpool_storage_address, address) do update set id = excluded.id, url = excluded.url, joined_time = excluded.joined_time, last_proposal_time = excluded.last_proposal_time, rpl_bond_amount = excluded.rpl_bond_amount, unbonded_validator_count = excluded.unbonded_validator_count`, strings.Join(valueStrings, ",")) + stmt := fmt.Sprintf(` + INSERT INTO rocketpool_dao_members ( + rocketpool_storage_address, + address, + id, + url, + joined_time, + last_proposal_time, + rpl_bond_amount, + unbonded_validator_count + ) + values %s + on conflict (rocketpool_storage_address, address) do update set + id = excluded.id, + url = excluded.url, + joined_time = excluded.joined_time, + last_proposal_time = excluded.last_proposal_time, + rpl_bond_amount = excluded.rpl_bond_amount, + unbonded_validator_count = excluded.unbonded_validator_count + `, strings.Join(valueStrings, ",")) _, err := tx.Exec(stmt, valueArgs...) if err != nil { return fmt.Errorf("error inserting into rocketpool_dao_members: %w", err) } + + _, err = tx.Exec(` + DELETE FROM rocketpool_dao_members + WHERE NOT address = ANY($1)`, pq.ByteaArray(addresses)) + if err != nil { + return fmt.Errorf("error deleting from rocketpool_dao_members: %w", err) + } } return tx.Commit() @@ -1858,7 +1887,7 @@ func DownloadRewardsFile(fileName string, interval uint64, cid string, isDaemon continue } else { // If we got here, we have a successful download - bytes, err := ioutil.ReadAll(resp.Body) + bytes, err := io.ReadAll(resp.Body) if err != nil { errBuilder.WriteString(fmt.Sprintf("Error reading response bytes from %s: %s\n", url, err.Error())) continue diff --git a/exporter/sync_committees.go b/exporter/sync_committees.go index 4f7c1d7a85..c76d049f44 100644 --- a/exporter/sync_committees.go +++ b/exporter/sync_committees.go @@ -86,17 +86,28 @@ func exportSyncCommitteeAtPeriod(rpcClient rpc.Client, p uint64) error { validatorsU64[i] = idxU64 } - start := time.Now() - firstSlot := firstEpoch * utils.Config.Chain.ClConfig.SlotsPerEpoch - lastSlot := lastEpoch*utils.Config.Chain.ClConfig.SlotsPerEpoch + utils.Config.Chain.ClConfig.SlotsPerEpoch - 1 - logger.Infof("exporting sync committee assignments for period %v (epoch %v to %v, slot %v to %v) to bigtable", p, firstEpoch, lastEpoch, firstSlot, lastSlot) + dedupMap := make(map[uint64]bool) - err = db.BigtableClient.SaveSyncCommitteesAssignments(firstSlot, lastSlot, validatorsU64) - if err != nil { - return fmt.Errorf("error saving sync committee assignments: %v", err) + for _, validator := range validatorsU64 { + dedupMap[validator] = true + } + + validatorsU64 = make([]uint64, 0, len(dedupMap)) + for validator := range dedupMap { + validatorsU64 = append(validatorsU64, validator) } - logger.Infof("exported sync committee assignments for period %v to bigtable in %v", p, time.Since(start)) + // start := time.Now() + // + // firstSlot := firstEpoch * utils.Config.Chain.ClConfig.SlotsPerEpoch + // lastSlot := lastEpoch*utils.Config.Chain.ClConfig.SlotsPerEpoch + utils.Config.Chain.ClConfig.SlotsPerEpoch - 1 + // logger.Infof("exporting sync committee assignments for period %v (epoch %v to %v, slot %v to %v) to bigtable", p, firstEpoch, lastEpoch, firstSlot, lastSlot) + + // err = db.BigtableClient.SaveSyncCommitteesAssignments(firstSlot, lastSlot, validatorsU64) + // if err != nil { + // return fmt.Errorf("error saving sync committee assignments: %v", err) + // } + // logger.Infof("exported sync committee assignments for period %v to bigtable in %v", p, time.Since(start)) tx, err := db.WriterDb.Beginx() if err != nil { return err @@ -104,8 +115,8 @@ func exportSyncCommitteeAtPeriod(rpcClient rpc.Client, p uint64) error { defer tx.Rollback() nArgs := 3 - valueArgs := make([]interface{}, len(c.Validators)*nArgs) - valueIds := make([]string, len(c.Validators)) + valueArgs := make([]interface{}, len(validatorsU64)*nArgs) + valueIds := make([]string, len(validatorsU64)) for i, idxU64 := range validatorsU64 { valueArgs[i*nArgs+0] = p valueArgs[i*nArgs+1] = idxU64 diff --git a/exporter/sync_committees_count.go b/exporter/sync_committees_count.go index 591254f3ca..0ff2c84502 100644 --- a/exporter/sync_committees_count.go +++ b/exporter/sync_committees_count.go @@ -26,8 +26,12 @@ func exportSyncCommitteesCount() error { return err } - currEpoch := utils.TimeToEpoch(time.Now()) - currentPeriod := utils.SyncPeriodOfEpoch(uint64(currEpoch)) + latestFinalizedEpoch, err := db.GetLatestFinalizedEpoch() + if err != nil { + logger.Errorf("error retrieving latest exported finalized epoch from the database: %v", err) + } + + currentPeriod := utils.SyncPeriodOfEpoch(latestFinalizedEpoch) firstPeriod := utils.SyncPeriodOfEpoch(utils.Config.Chain.ClConfig.AltairForkEpoch) dbPeriod := uint64(0) @@ -73,7 +77,7 @@ func exportSyncCommitteesCountAtPeriod(period uint64, countSoFar float64) (float totalValidatorsCount := uint64(0) err := db.WriterDb.Get(&totalValidatorsCount, "SELECT validatorscount FROM epochs WHERE epoch = $1", e) if err != nil { - return 0, err + return 0, fmt.Errorf("error retrieving validatorscount for epoch %v: %v", e, err) } count = countSoFar + (float64(utils.Config.Chain.ClConfig.SyncCommitteeSize) / float64(totalValidatorsCount)) } diff --git a/go.mod b/go.mod index 0f19e55341..ce1c959fbb 100644 --- a/go.mod +++ b/go.mod @@ -14,10 +14,10 @@ require ( github.com/aws/aws-sdk-go-v2 v1.21.0 github.com/aws/aws-sdk-go-v2/credentials v1.1.1 github.com/aws/aws-sdk-go-v2/service/s3 v1.38.5 - github.com/aws/smithy-go v1.14.2 github.com/aybabtme/uniplot v0.0.0-20151203143629-039c559e5e7e + github.com/carlmjohnson/requests v0.23.4 github.com/davecgh/go-spew v1.1.1 - github.com/ethereum/go-ethereum v1.12.2-0.20230824084913-1a2135044cd4 + github.com/ethereum/go-ethereum v1.12.2 github.com/evanw/esbuild v0.8.23 github.com/go-redis/redis/v8 v8.11.5 github.com/gobitfly/eth-rewards v0.1.2-0.20230403064929-411ddc40a5f7 @@ -47,7 +47,7 @@ require ( github.com/pkg/errors v0.9.1 github.com/pressly/goose/v3 v3.10.0 github.com/prometheus/client_golang v1.14.0 - github.com/protolambda/zrnt v0.12.4 + github.com/protolambda/zrnt v0.30.0 github.com/prysmaticlabs/go-bitfield v0.0.0-20210809151128-385d8c5e3fb7 github.com/prysmaticlabs/go-ssz v0.0.0-20210121151755-f6208871c388 github.com/prysmaticlabs/prysm/v3 v3.2.0 @@ -60,13 +60,13 @@ require ( github.com/swaggo/http-swagger v1.3.0 github.com/swaggo/swag v1.8.3 github.com/urfave/negroni v1.0.0 - github.com/wealdtech/go-ens/v3 v3.5.5 + github.com/wealdtech/go-ens/v3 v3.6.0 github.com/wealdtech/go-eth2-types/v2 v2.8.1 github.com/wealdtech/go-eth2-util v1.8.1 github.com/zesik/proxyaddr v0.0.0-20161218060608-ec32c535184d - golang.org/x/crypto v0.12.0 + golang.org/x/crypto v0.13.0 golang.org/x/sync v0.3.0 - golang.org/x/text v0.12.0 + golang.org/x/text v0.13.0 google.golang.org/api v0.102.0 google.golang.org/protobuf v1.28.1 gopkg.in/yaml.v3 v3.0.1 @@ -75,7 +75,7 @@ require ( require ( github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect github.com/donovanhide/eventsource v0.0.0-20210830082556-c59027999da0 - golang.org/x/exp v0.0.0-20230810033253-352e893a4cad + golang.org/x/exp v0.0.0-20230905200255-921286631fa9 ) require ( @@ -84,8 +84,10 @@ require ( cloud.google.com/go/compute/metadata v0.2.1 // indirect cloud.google.com/go/iam v0.7.0 // indirect cloud.google.com/go/longrunning v0.3.0 // indirect - github.com/Microsoft/go-winio v0.6.1 // indirect + github.com/DataDog/zstd v1.5.2 // indirect + github.com/VictoriaMetrics/fastcache v1.6.0 // indirect github.com/alessio/shellescape v1.4.1 // indirect + github.com/allegro/bigcache v1.2.1 // indirect github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.13 // indirect github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.41 // indirect github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.35 // indirect @@ -94,36 +96,52 @@ require ( github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.36 // indirect github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.35 // indirect github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.15.4 // indirect + github.com/aws/smithy-go v1.14.2 // indirect github.com/bits-and-blooms/bitset v1.7.0 // indirect github.com/census-instrumentation/opencensus-proto v0.2.1 // indirect github.com/cncf/udpa/go v0.0.0-20210930031921-04548b0d99d4 // indirect github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1 // indirect + github.com/cockroachdb/errors v1.9.1 // indirect + github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b // indirect + github.com/cockroachdb/pebble v0.0.0-20230209160836-829675f94811 // indirect + github.com/cockroachdb/redact v1.1.3 // indirect github.com/consensys/bavard v0.1.13 // indirect github.com/consensys/gnark-crypto v0.10.0 // indirect github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect github.com/crate-crypto/go-kzg-4844 v0.3.0 // indirect - github.com/deckarep/golang-set/v2 v2.1.0 // indirect + github.com/deckarep/golang-set/v2 v2.3.1 // indirect github.com/dgraph-io/ristretto v0.1.1 // indirect github.com/dustin/go-humanize v1.0.1 // indirect github.com/envoyproxy/go-control-plane v0.10.2-0.20220325020618-49ff273808a1 // indirect github.com/envoyproxy/protoc-gen-validate v0.1.0 // indirect github.com/ethereum/c-kzg-4844 v0.3.1 // indirect github.com/fsnotify/fsnotify v1.6.0 // indirect + github.com/getsentry/sentry-go v0.18.0 // indirect + github.com/gofrs/flock v0.8.1 // indirect + github.com/gogo/protobuf v1.3.2 // indirect github.com/googleapis/enterprise-certificate-proxy v0.2.0 // indirect + github.com/holiman/bloomfilter/v2 v2.0.3 // indirect github.com/holiman/uint256 v1.2.3 // indirect github.com/imdario/mergo v0.3.13 // indirect - github.com/ipfs/go-cid v0.3.2 // indirect + github.com/ipfs/go-cid v0.4.1 // indirect + github.com/kr/pretty v0.3.1 // indirect + github.com/kr/text v0.2.0 // indirect + github.com/mattn/go-runewidth v0.0.14 // indirect + github.com/mattn/go-sqlite3 v1.11.0 // indirect github.com/mitchellh/go-homedir v1.1.0 // indirect github.com/mmcloughlin/addchain v0.4.0 // indirect github.com/multiformats/go-base32 v0.1.0 // indirect github.com/multiformats/go-base36 v0.2.0 // indirect - github.com/multiformats/go-multibase v0.1.1 // indirect - github.com/multiformats/go-multihash v0.2.1 // indirect + github.com/multiformats/go-multibase v0.2.0 // indirect + github.com/multiformats/go-multihash v0.2.3 // indirect github.com/multiformats/go-varint v0.0.7 // indirect + github.com/olekukonko/tablewriter v0.0.5 // indirect github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 // indirect github.com/protolambda/zssz v0.1.5 // indirect github.com/prysmaticlabs/fastssz v0.0.0-20221107182844-78142813af44 // indirect github.com/prysmaticlabs/gohashtree v0.0.2-alpha // indirect + github.com/rivo/uniseg v0.4.3 // indirect + github.com/rogpeppe/go-internal v1.9.0 // indirect github.com/rs/cors v1.8.0 // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect github.com/spaolacci/murmur3 v1.1.0 // indirect @@ -132,10 +150,10 @@ require ( github.com/wealdtech/go-merkletree v1.0.1-0.20190605192610-2bb163c2ea2a // indirect github.com/wealdtech/go-multicodec v1.4.0 // indirect github.com/yusufpapurcu/wmi v1.2.2 // indirect - golang.org/x/mod v0.11.0 // indirect google.golang.org/genproto v0.0.0-20221118155620-16455021b5e6 // indirect google.golang.org/grpc v1.52.3 // indirect - lukechampine.com/blake3 v1.1.7 // indirect + gopkg.in/natefinch/npipe.v2 v2.0.0-20160621034901-c1b8fa8bdcce // indirect + lukechampine.com/blake3 v1.2.1 // indirect rsc.io/tmplfunc v0.0.3 // indirect ) @@ -153,7 +171,7 @@ require ( github.com/fatih/color v1.14.1 // indirect github.com/ferranbt/fastssz v0.1.3 // indirect github.com/go-chi/chi v4.0.2+incompatible // indirect - github.com/go-ole/go-ole v1.2.6 // indirect + github.com/go-ole/go-ole v1.3.0 // indirect github.com/go-openapi/jsonpointer v0.19.5 // indirect github.com/go-openapi/jsonreference v0.20.0 // indirect github.com/go-openapi/spec v0.20.6 // indirect @@ -164,7 +182,7 @@ require ( github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/snappy v0.0.5-0.20220116011046-fa5810519dcb // indirect github.com/google/go-cmp v0.5.9 // indirect - github.com/google/uuid v1.3.0 + github.com/google/uuid v1.3.1 github.com/googleapis/gax-go/v2 v2.6.0 // indirect github.com/gorilla/securecookie v1.1.1 // indirect github.com/grpc-ecosystem/grpc-gateway/v2 v2.11.3 // indirect @@ -180,13 +198,13 @@ require ( github.com/jackc/puddle v1.3.0 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/klauspost/compress v1.16.0 - github.com/klauspost/cpuid/v2 v2.2.4 // indirect + github.com/klauspost/cpuid/v2 v2.2.5 // indirect github.com/mailru/easyjson v0.7.7 // indirect github.com/mattn/go-colorable v0.1.13 // indirect github.com/mattn/go-isatty v0.0.17 // indirect github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect github.com/minio/highwayhash v1.0.2 // indirect - github.com/minio/sha256-simd v1.0.0 // indirect + github.com/minio/sha256-simd v1.0.1 // indirect github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 // indirect github.com/patrickmn/go-cache v2.1.0+incompatible github.com/prometheus/client_model v0.3.0 // indirect @@ -199,13 +217,13 @@ require ( github.com/swaggo/files v0.0.0-20220610200504-28940afbdbfe // indirect github.com/syndtr/goleveldb v1.0.1-0.20220721030215-126854af5e6d github.com/thomaso-mirodin/intmath v0.0.0-20160323211736-5dc6d854e46e // indirect - github.com/tklauser/go-sysconf v0.3.11 // indirect - github.com/tklauser/numcpus v0.6.0 // indirect + github.com/tklauser/go-sysconf v0.3.12 // indirect + github.com/tklauser/numcpus v0.6.1 // indirect go.opencensus.io v0.24.0 // indirect - golang.org/x/net v0.10.0 // indirect + golang.org/x/net v0.15.0 // indirect golang.org/x/oauth2 v0.3.0 // indirect - golang.org/x/sys v0.11.0 // indirect - golang.org/x/tools v0.9.1 // indirect + golang.org/x/sys v0.12.0 // indirect + golang.org/x/tools v0.13.0 // indirect golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect gonum.org/v1/gonum v0.12.0 // indirect google.golang.org/appengine v1.6.7 // indirect diff --git a/go.sum b/go.sum index 101154f0dd..63ddf43d3e 100644 --- a/go.sum +++ b/go.sum @@ -55,21 +55,27 @@ contrib.go.opencensus.io/exporter/jaeger v0.2.1 h1:yGBYzYMewVL0yO9qqJv3Z5+IRhPdU dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= firebase.google.com/go v3.13.0+incompatible h1:3TdYC3DDi6aHn20qoRkxwGqNgdjtblwVAyRLQwGn/+4= firebase.google.com/go v3.13.0+incompatible/go.mod h1:xlah6XbEyW6tbfSklcfe5FHJIwjt8toICdV5Wh9ptHs= +github.com/AndreasBriese/bbloom v0.0.0-20190306092124-e2d15f34fcf9/go.mod h1:bOvUY6CB00SOBii9/FifXqc0awNKxLFCL/+pkDPuyl8= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/toml v1.2.1 h1:9F2/+DoOYIOksmaJFPw1tGFy1eDnIJXg+UHjuD8lTak= github.com/BurntSushi/toml v1.2.1/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= +github.com/CloudyKit/fastprinter v0.0.0-20200109182630-33d98a066a53/go.mod h1:+3IMCy2vIlbG1XG/0ggNQv0SvxCAIpPM5b1nCz56Xno= +github.com/CloudyKit/jet/v3 v3.0.0/go.mod h1:HKQPgSJmdK8hdoAbKUUWajkHyHo4RaU5rMdUywE7VMo= github.com/DataDog/zstd v1.5.2 h1:vUG4lAyuPCXO0TLbXvPv7EB7cNK1QV/luu55UHLrrn8= +github.com/DataDog/zstd v1.5.2/go.mod h1:g4AWEaM3yOg3HYfnJ3YIawPnVdXJh9QME85blwSAmyw= github.com/Gurpartap/storekit-go v0.0.0-20201205024111-36b6cd5c6a21 h1:HcdvlzaQ4CJfH7xbfJZ3ZHN//BTEpId46iKEMuP3wHE= github.com/Gurpartap/storekit-go v0.0.0-20201205024111-36b6cd5c6a21/go.mod h1:7PODFS++oNZ6khojmPBvkrDeFO/hrc3jmvWvQAOXorw= +github.com/Joker/hpp v1.0.0/go.mod h1:8x5n+M1Hp5hC0g8okX3sR3vFQwynaX/UgSOM9MeBKzY= github.com/KyleBanks/depth v1.2.1 h1:5h8fQADFrWtarTdtDudMmGsC7GPbOAu6RVB3ffsVFHc= github.com/KyleBanks/depth v1.2.1/go.mod h1:jzSb9d0L43HxTQfT+oSA1EEp2q+ne2uh6XgeJcm8brE= github.com/Masterminds/semver/v3 v3.1.1 h1:hLg3sBzpNErnxhQtUy/mmLR2I9foDujNK030IGemrRc= github.com/Masterminds/semver/v3 v3.1.1/go.mod h1:VPu/7SZ7ePZ3QOrcuXROw5FAcLl4a0cBrbBpGY/8hQs= -github.com/Microsoft/go-winio v0.6.1 h1:9/kr64B9VUZrLm5YYwbGtUJnMgqWVOdUAXu6Migciow= -github.com/Microsoft/go-winio v0.6.1/go.mod h1:LRdKpFKfdobln8UmuiYcKPot9D2v6svN5+sAH+4kjUM= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= +github.com/Shopify/goreferrer v0.0.0-20181106222321-ec9c9a553398/go.mod h1:a1uqRtAwp2Xwc6WNPJEufxJ7fx3npB4UV/JOLmbu5I0= github.com/VictoriaMetrics/fastcache v1.6.0 h1:C/3Oi3EiBCqufydp1neRZkqcwmEiuRT9c3fqvvgKm5o= +github.com/VictoriaMetrics/fastcache v1.6.0/go.mod h1:0qHz5QP0GMX4pfmMA/zt5RgfNuXJrTP0zS7DqpHGGTw= +github.com/ajg/form v1.5.1/go.mod h1:uL1WgH+h2mgNtvBq0339dVnzXdBETtL2LeUXaIv25UY= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= @@ -81,8 +87,12 @@ github.com/alexedwards/scs/redisstore v0.0.0-20230217120314-6b1bedc0f08c h1:m2XN github.com/alexedwards/scs/redisstore v0.0.0-20230217120314-6b1bedc0f08c/go.mod h1:ceKFatoD+hfHWWeHOAYue1J+XgOJjE7dw8l3JtIRTGY= github.com/alexedwards/scs/v2 v2.5.0 h1:zgxOfNFmiJyXG7UPIuw1g2b9LWBeRLh3PjfB9BDmfL4= github.com/alexedwards/scs/v2 v2.5.0/go.mod h1:ToaROZxyKukJKT/xLcVQAChi5k6+Pn1Gvmdl7h3RRj8= +github.com/allegro/bigcache v1.2.1-0.20190218064605-e24eb225f156/go.mod h1:Cb/ax3seSYIx7SuZdm2G2xzfwmv3TPSk2ucNfQESPXM= +github.com/allegro/bigcache v1.2.1 h1:hg1sY1raCwic3Vnsvje6TT7/pnZba83LeFck5NrFKSc= +github.com/allegro/bigcache v1.2.1/go.mod h1:Cb/ax3seSYIx7SuZdm2G2xzfwmv3TPSk2ucNfQESPXM= github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= +github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= github.com/asaskevich/govalidator v0.0.0-20210307081110-f21760c49a8d h1:Byv0BzEl3/e6D5CLfI0j/7hiIEtvGVFPCZ7Ei2oq8iQ= @@ -123,6 +133,7 @@ github.com/aws/smithy-go v1.14.2 h1:MJU9hqBGbvWZdApzpvoF2WAIJDbtjK2NDJSiJP7HblQ= github.com/aws/smithy-go v1.14.2/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J3gBHzA= github.com/aybabtme/uniplot v0.0.0-20151203143629-039c559e5e7e h1:dSeuFcs4WAJJnswS8vXy7YY1+fdlbVPuEVmDAfqvFOQ= github.com/aybabtme/uniplot v0.0.0-20151203143629-039c559e5e7e/go.mod h1:uh71c5Vc3VNIplXOFXsnDy21T1BepgT32c5X/YPrOyc= +github.com/aymerick/raymond v2.0.3-0.20180322193309-b565731e1464+incompatible/go.mod h1:osfaiScAUVup+UC9Nfq76eWqDhXlp+4UYaA8uhTBO6g= github.com/bazelbuild/rules_go v0.23.2 h1:Wxu7JjqnF78cKZbsBsARLSXx/jlGaSLCnUV3mTlyHvM= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= @@ -140,6 +151,8 @@ github.com/btcsuite/btcd/btcec/v2 v2.3.2 h1:5n0X6hX0Zk+6omWcihdYvdAlGf2DfasC0GMf github.com/btcsuite/btcd/btcec/v2 v2.3.2/go.mod h1:zYzJ8etWJQIv1Ogk7OzpWjowwOdXY1W/17j2MW85J04= github.com/btcsuite/btcd/chaincfg/chainhash v1.0.1 h1:q0rUy8C/TYNBQS1+CGKw68tLOFYSNEs0TFnxxnS9+4U= github.com/bufbuild/buf v0.37.0/go.mod h1:lQ1m2HkIaGOFba6w/aC3KYBHhKEOESP3gaAEpS3dAFM= +github.com/carlmjohnson/requests v0.23.4 h1:AxcvapfB9RPXLSyvAHk9YJoodQ43ZjzNHj6Ft3tQGdg= +github.com/carlmjohnson/requests v0.23.4/go.mod h1:Qzp6tW4DQyainPP+tGwiJTzwxvElTIKm0B191TgTtOA= github.com/census-instrumentation/opencensus-proto v0.2.1 h1:glEXhBS5PSLLv4IXzLA5yPRVX4bilULVyxxbrfOtDAk= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/cp v1.1.1 h1:nCb6ZLdB7NRaqsm91JtQTAme2SKJzXVsdPIPkyJr1MU= @@ -163,10 +176,18 @@ github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1 h1:zH8ljVhhq7yC0MIeUL/ github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cockroachdb/apd v1.1.0 h1:3LFP3629v+1aKXU5Q37mxmRxX/pIu1nijXydLShEq5I= github.com/cockroachdb/apd v1.1.0/go.mod h1:8Sl8LxpKi29FqWXR16WEFZRNSz3SoPzUzeMeY4+DwBQ= +github.com/cockroachdb/datadriven v1.0.2 h1:H9MtNqVoVhvd9nCBwOyDjUEdZCREqbIdCJD93PBm/jA= +github.com/cockroachdb/datadriven v1.0.2/go.mod h1:a9RdTaap04u637JoCzcUoIcDmvwSUtcUFtT/C3kJlTU= github.com/cockroachdb/errors v1.9.1 h1:yFVvsI0VxmRShfawbt/laCIDy/mtTqqnvoNgiy5bEV8= +github.com/cockroachdb/errors v1.9.1/go.mod h1:2sxOtL2WIc096WSZqZ5h8fa17rdDq9HZOZLBCor4mBk= +github.com/cockroachdb/logtags v0.0.0-20211118104740-dabe8e521a4f/go.mod h1:Vz9DsVWQQhf3vs21MhPMZpMGSht7O/2vFW2xusFUVOs= github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b h1:r6VH0faHjZeQy818SGhaone5OnYfxFR/+AzdY3sf5aE= -github.com/cockroachdb/pebble v0.0.0-20230821143352-55b44ac08de8 h1:2fNOyQqrHSjFdqnKqSLj9W7eGTcqKujJPA8vAuT1zN8= +github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b/go.mod h1:Vz9DsVWQQhf3vs21MhPMZpMGSht7O/2vFW2xusFUVOs= +github.com/cockroachdb/pebble v0.0.0-20230209160836-829675f94811 h1:ytcWPaNPhNoGMWEhDvS3zToKcDpRsLuRolQJBVGdozk= +github.com/cockroachdb/pebble v0.0.0-20230209160836-829675f94811/go.mod h1:Nb5lgvnQ2+oGlE/EyZy4+2/CxRh9KfvCXnag1vtpxVM= github.com/cockroachdb/redact v1.1.3 h1:AKZds10rFSIj7qADf0g46UixK8NNLwWTNdCIGS5wfSQ= +github.com/cockroachdb/redact v1.1.3/go.mod h1:BVNblN9mBWFyMyqK1k3AAiSxhvhfK2oOZZ2lK+dpvRg= +github.com/codegangsta/inject v0.0.0-20150114235600-33e0aa1cb7c0/go.mod h1:4Zcjuz89kmFXt9morQgcfYZAYZ5n8WHjt81YYWIwtTM= github.com/consensys/bavard v0.1.13 h1:oLhMLOFGTLdlda/kma4VOJazblc7IM5y5QPd2A/YjhQ= github.com/consensys/bavard v0.1.13/go.mod h1:9ItSMtA/dXMAiL7BG6bqW2m3NdSEObYWoH223nGHukI= github.com/consensys/gnark-crypto v0.10.0 h1:zRh22SR7o4K35SoNqouS9J/TKHTyU2QWaj5ldehyXtA= @@ -174,12 +195,16 @@ github.com/consensys/gnark-crypto v0.10.0/go.mod h1:Iq/P3HHl0ElSjsg2E1gsMwhAyxnx github.com/coocood/freecache v1.2.3 h1:lcBwpZrwBZRZyLk/8EMyQVXRiFl663cCuMOrjCALeto= github.com/coocood/freecache v1.2.3/go.mod h1:RBUWa/Cy+OHdfTGFEhEuE1pMCMX51Ncizj7rthiQ3vk= github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= +github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= +github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk= +github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/go-systemd v0.0.0-20190719114852-fd7a80b32e1f/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= +github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE= github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/cpuguy83/go-md2man/v2 v2.0.2 h1:p1EgwI/C7NhT0JmVkwCD2ZBK8j4aeHQX2pMHHBfMQ6w= github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= @@ -191,11 +216,12 @@ github.com/d4l3k/messagediff v1.2.1 h1:ZcAIMYsUg0EAp9X+tt8/enBE/Q8Yd5kzPynLyKptt github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/deckarep/golang-set/v2 v2.1.0 h1:g47V4Or+DUdzbs8FxCCmgb6VYd+ptPAngjM6dtGktsI= -github.com/deckarep/golang-set/v2 v2.1.0/go.mod h1:VAky9rY/yGXJOLEDv3OMci+7wtDpOF4IN+y82NBOac4= +github.com/deckarep/golang-set/v2 v2.3.1 h1:vjmkvJt/IV27WXPyYQpAh4bRyWJc5Y435D17XQ9QU5A= +github.com/deckarep/golang-set/v2 v2.3.1/go.mod h1:VAky9rY/yGXJOLEDv3OMci+7wtDpOF4IN+y82NBOac4= github.com/decred/dcrd/crypto/blake256 v1.0.1 h1:7PltbUIQB7u/FfZ39+DGa/ShuMyJ5ilcvdfma9wOH6Y= github.com/decred/dcrd/dcrec/secp256k1/v4 v4.2.0 h1:8UrgZ3GkP4i/CLijOJx79Yu+etlyjdBU4sfcs2WYQMs= github.com/decred/dcrd/dcrec/secp256k1/v4 v4.2.0/go.mod h1:v57UDF4pDQJcEfFUCRop3lJL149eHGSe9Jvczhzjo/0= +github.com/dgraph-io/badger v1.6.0/go.mod h1:zwt7syl517jmP8s94KqSxTlM6IMsdhYy6psNgSztDR4= github.com/dgraph-io/ristretto v0.1.1 h1:6CWw5tJNgpegArSHpNHJKldNeq03FQCwYvfMVWajOK8= github.com/dgraph-io/ristretto v0.1.1/go.mod h1:S1GPSBCYCIhmVNfcth17y2zZtQT6wzkzgwUve0VDWWA= github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= @@ -209,6 +235,7 @@ github.com/donovanhide/eventsource v0.0.0-20210830082556-c59027999da0/go.mod h1: github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= +github.com/eknkc/amber v0.0.0-20171010120322-cdade1c07385/go.mod h1:0vRUJqYpeSZifjYj7uP3BG/gKcuzL9xWVV/Y+cK33KM= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= @@ -219,10 +246,11 @@ github.com/envoyproxy/go-control-plane v0.10.2-0.20220325020618-49ff273808a1 h1: github.com/envoyproxy/go-control-plane v0.10.2-0.20220325020618-49ff273808a1/go.mod h1:KJwIaB5Mv44NWtYuAOFCVOjcI94vtpEz2JU/D2v6IjE= github.com/envoyproxy/protoc-gen-validate v0.1.0 h1:EQciDnbrYxy13PgWoY8AqoxGiPrpgBZ1R8UNe3ddc+A= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= +github.com/etcd-io/bbolt v1.3.3/go.mod h1:ZF2nL25h33cCyBtcyWeZ2/I3HQOfTP+0PIEvHjkjCrw= github.com/ethereum/c-kzg-4844 v0.3.1 h1:sR65+68+WdnMKxseNWxSJuAv2tsUrihTpVBTfM/U5Zg= github.com/ethereum/c-kzg-4844 v0.3.1/go.mod h1:VewdlzQmpT5QSrVhbBuGoCdFJkpaJlO1aQputP83wc0= -github.com/ethereum/go-ethereum v1.12.2-0.20230824084913-1a2135044cd4 h1:hdv9mGg6nirDb5f+1bClbjqKxLNYhgZw+fJHU71dcZE= -github.com/ethereum/go-ethereum v1.12.2-0.20230824084913-1a2135044cd4/go.mod h1:UWQqzN9473khWbdvlh2QjYXX0ykhKT1OuPYus9Hgv+o= +github.com/ethereum/go-ethereum v1.12.2 h1:eGHJ4ij7oyVqUQn48LBz3B7pvQ8sV0wGJiIE6gDq/6Y= +github.com/ethereum/go-ethereum v1.12.2/go.mod h1:1cRAEV+rp/xX0zraSCBnu9Py3HQ+geRMj3HdR+k0wfI= github.com/evanw/esbuild v0.8.23 h1:eRRG1fNtQ9KPG3lM62EUYagLVMSuxSTBEgukqY0et3w= github.com/evanw/esbuild v0.8.23/go.mod h1:y2AFBAGVelPqPodpdtxWWqe6n2jYf5FrsJbligmRmuw= github.com/facebookgo/ensure v0.0.0-20160127193407-b4ab57deab51 h1:0JZ+dUmQeA8IIVUMzysrX4/AKuQwWhV2dYQuPZdvdSQ= @@ -231,11 +259,13 @@ github.com/facebookgo/stack v0.0.0-20160209184415-751773369052 h1:JWuenKqqX8nojt github.com/facebookgo/stack v0.0.0-20160209184415-751773369052/go.mod h1:UbMTZqLaRiH3MsBH8va0n7s1pQYcu3uTb8G4tygF4Zg= github.com/facebookgo/subset v0.0.0-20150612182917-8dac2c3c4870 h1:E2s37DuLxFhQDg5gKsWoLBOB0n+ZW8s599zru8FJ2/Y= github.com/facebookgo/subset v0.0.0-20150612182917-8dac2c3c4870/go.mod h1:5tD+neXqOorC30/tWg0LCSkrqj/AR6gu8yY8/fpw1q0= +github.com/fasthttp-contrib/websocket v0.0.0-20160511215533-1f3b11f56072/go.mod h1:duJ4Jxv5lDcvg4QuQr0oowTf7dz4/CR8NtyCooz9HL8= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/fatih/color v1.10.0/go.mod h1:ELkj/draVOlAH/xkhN6mQ50Qd0MPOk5AAr3maGEBuJM= github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk= github.com/fatih/color v1.14.1 h1:qfhVLaG5s+nCROl1zJsZRxFeYrHLqWroPOQ8BWiNb4w= github.com/fatih/color v1.14.1/go.mod h1:2oHN61fhTpgcxD3TSWCgKDiH1+x4OiDVVGH8WlgGZGg= +github.com/fatih/structs v1.1.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M= github.com/ferranbt/fastssz v0.1.2/go.mod h1:X5UPrE2u1UJjxHA8X54u04SBwdAQjG2sFtWs39YxyWs= github.com/ferranbt/fastssz v0.1.3 h1:ZI+z3JH05h4kgmFXdHuR1aWYsgrg7o+Fw7/NCzM16Mo= github.com/ferranbt/fastssz v0.1.3/go.mod h1:0Y9TEd/9XuFlh7mskMPfXiI2Dkw4Ddg9EyXt1W7MRvE= @@ -245,14 +275,22 @@ github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4 github.com/fsnotify/fsnotify v1.5.4/go.mod h1:OVB6XrOHzAwXMpEM7uPOzcehqUV2UqJxmVXmkdnm1bU= github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY= github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw= +github.com/gavv/httpexpect v2.0.0+incompatible/go.mod h1:x+9tiU1YnrOvnB725RkpoLv1M62hOWzwo5OXotisrKc= github.com/gballet/go-libpcsclite v0.0.0-20191108122812-4678299bea08 h1:f6D9Hr8xV8uYKlyuj8XIruxlh9WjVjdh1gIicAS7ays= +github.com/getsentry/sentry-go v0.12.0/go.mod h1:NSap0JBYWzHND8oMbyi0+XZhUalc1TBdRL1M71JZW2c= github.com/getsentry/sentry-go v0.18.0 h1:MtBW5H9QgdcJabtZcuJG80BMOwaBpkRDZkxRkNC1sN0= +github.com/getsentry/sentry-go v0.18.0/go.mod h1:Kgon4Mby+FJ7ZWHFUAZgVaIa8sxHtnRJRLTXZr51aKQ= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= +github.com/gin-contrib/sse v0.0.0-20190301062529-5545eab6dad3/go.mod h1:VJ0WA2NBN22VlZ2dKZQPAPnyWw5XTlK1KymzLKsr59s= github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm+fLHvGI= +github.com/gin-gonic/gin v1.4.0/go.mod h1:OW2EZn3DO8Ln9oIKOvM++LBO+5UPHJJDH72/q/3rZdM= github.com/gin-gonic/gin v1.5.0/go.mod h1:Nd6IXA8m5kNZdNEHMBd93KT+mdY3+bewLgRvmCsR2Do= +github.com/go-check/check v0.0.0-20180628173108-788fd7840127/go.mod h1:9ES+weclKsC9YodN5RgxqK/VD9HM9JsCSh7rNhMZE98= github.com/go-chi/chi v4.0.0+incompatible/go.mod h1:eB3wogJHnLi3x/kFX2A+IbTBlXxmMeXJVKy9tTv1XzQ= github.com/go-chi/chi v4.0.2+incompatible h1:maB6vn6FqCxrpz4FqWdh4+lwpyZIQS7YEAUcHlgXVRs= github.com/go-chi/chi v4.0.2+incompatible/go.mod h1:eB3wogJHnLi3x/kFX2A+IbTBlXxmMeXJVKy9tTv1XzQ= +github.com/go-errors/errors v1.0.1/go.mod h1:f4zRHt4oKfwPJE5k8C9vpYG+aDHdBFUsgrm6/TyX73Q= +github.com/go-errors/errors v1.4.2 h1:J6MZopCL4uSllY1OfXM374weqZFFItUbrImctkmUxIA= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= @@ -263,8 +301,10 @@ github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9 github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= github.com/go-logr/logr v1.2.3 h1:2DntVwHkVopvECVRSlL5PSo9eG+cAkDCuckLubN+rq0= -github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY= +github.com/go-martini/martini v0.0.0-20170121215854-22fa46961aab/go.mod h1:/P9AEU963A2AYjv4d1V5eVL1CQbEJq6aCNHDDjibzu8= github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= +github.com/go-ole/go-ole v1.3.0 h1:Dt6ye7+vXGIKZ7Xtk4s6/xVdGDQynvom7xCFEdWr6uE= +github.com/go-ole/go-ole v1.3.0/go.mod h1:5LS6F96DhAwUc7C+1HLexzMXY1xGRSryjyPPKW6zv78= github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= github.com/go-openapi/jsonpointer v0.19.5 h1:gZr+CIYByUqjcgeLXnQu2gHYQC9o73G2XUeOFYEICuY= github.com/go-openapi/jsonpointer v0.19.5/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= @@ -284,7 +324,7 @@ github.com/go-playground/universal-translator v0.16.0/go.mod h1:1AnU7NaIRDWWzGEK github.com/go-playground/universal-translator v0.17.0/go.mod h1:UkSxE5sNxxRwHyU+Scu5vgOQjsIJAF8j9muTVoKLVtA= github.com/go-playground/universal-translator v0.18.0 h1:82dyy6p4OuJq4/CByFNOn/jYrnRPArHwAcmLoJZxyho= github.com/go-playground/validator/v10 v10.4.1/go.mod h1:nlOn6nFhuKACm19sB/8EGNn9GlaMV7XkbRSipzJ0Ii4= -github.com/go-playground/validator/v10 v10.10.0 h1:I7mrTYv78z8k8VXa/qJlOlEXn/nBh+BF8dHX5nt/dr0= +github.com/go-playground/validator/v10 v10.11.1 h1:prmOlTVv+YjZjmRmNSF3VmspqJIxJWXmqUsHwfTRRkQ= github.com/go-redis/redis/v8 v8.11.5 h1:AcZZR7igkdvfVmQTPnu9WE37LRrO/YrBH5zWyjDC0oI= github.com/go-redis/redis/v8 v8.11.5/go.mod h1:gREzHqY1hg6oD9ngVRbLStwAWKhA0FEgq8Jd4h5lpwo= github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= @@ -299,17 +339,26 @@ github.com/gobitfly/eth.store v0.0.0-20230306141701-814b59fb0cea h1:r+d3DHTGauct github.com/gobitfly/eth.store v0.0.0-20230306141701-814b59fb0cea/go.mod h1:HAblwtMrQPJUfJDHwLLoa7RhLqGwinVyq56cKS34Hec= github.com/gobitfly/prysm/v3 v3.0.0-20230216184552-2f3f1e8190d5 h1:8kVoXCPhDwSjaGlKzBVQeE8n49k6jZumBGiP26FHNy0= github.com/gobitfly/prysm/v3 v3.0.0-20230216184552-2f3f1e8190d5/go.mod h1:+v+em7rOykPs93APGWCX/95/3uxU8bSVmbZ4+YNJzdA= +github.com/gobwas/httphead v0.0.0-20180130184737-2c6c146eadee/go.mod h1:L0fX3K22YWvt/FAX9NnzrNzcI4wNYi9Yku4O0LKYflo= +github.com/gobwas/pool v0.2.0/go.mod h1:q8bcK0KcYlCgd9e7WYLm9LpyS+YeLd8JVDW6WezmKEw= +github.com/gobwas/ws v1.0.2/go.mod h1:szmBTxLgaFppYjEmNtny/v3w89xOydFnnZMcgRRu/EM= github.com/goccy/go-yaml v1.9.2/go.mod h1:U/jl18uSupI5rdI2jmuCswEA2htH9eXfferR3KfscvA= github.com/goccy/go-yaml v1.10.0 h1:rBi+5HGuznOxx0JZ+60LDY85gc0dyIJCIMvsMJTKSKQ= github.com/goccy/go-yaml v1.10.0/go.mod h1:h/18Lr6oSQ3mvmqFoWmQ47KChOgpfHpTyIHl3yVmpiY= github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/gofrs/flock v0.8.0/go.mod h1:F1TvTiK9OcQqauNUHlbJvyl9Qa1QvF/gOUDKA14jxHU= github.com/gofrs/flock v0.8.1 h1:+gYjHKf32LDeiEEFhQaotPbLuUXjY5ZqxKgXy7n59aw= +github.com/gofrs/flock v0.8.1/go.mod h1:F1TvTiK9OcQqauNUHlbJvyl9Qa1QvF/gOUDKA14jxHU= github.com/gofrs/uuid v4.0.0+incompatible h1:1SD/1F5pU8p29ybwgQSwpQk+mwdRrXCYuPhW6m+TnJw= github.com/gofrs/uuid v4.0.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= +github.com/gogo/googleapis v0.0.0-20180223154316-0cd9801be74a/go.mod h1:gf4bu3Q80BeJ6H1S1vYPm8/ELATdvryBaNFGgqEef3s= +github.com/gogo/googleapis v1.4.1/go.mod h1:2lpHqI5OcWCtVElxXnPt+s8oJvMpySlOyM6xDCrzib4= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= +github.com/gogo/protobuf v1.2.0/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= +github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/gogo/status v1.1.0/go.mod h1:BFv9nrluPLmrS0EmGVvLaPNmRosr9KapBYd5/hpY1WM= github.com/golang-jwt/jwt v3.2.2+incompatible h1:IfV12K8xAKAnZqdXVzCZ+TOjboZ2keLg81eXfW3O+oY= github.com/golang-jwt/jwt v3.2.2+incompatible/go.mod h1:8pz2t5EyA70fFQQSrl6XZXzqecmYZeUEB8OUGHkxJ+I= github.com/golang-jwt/jwt/v4 v4.3.0 h1:kHL1vqdqWNfATmA0FNMdmZNMyZI1U6O31X4rlIPoBog= @@ -350,6 +399,7 @@ github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEW github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.5-0.20220116011046-fa5810519dcb h1:PBC98N2aIaM3XXiurYmW7fx4GZkL8feAMVq7nEjURHk= github.com/golang/snappy v0.0.5-0.20220116011046-fa5810519dcb/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/gomodule/redigo v1.7.1-0.20190724094224-574c33c3df38/go.mod h1:B4C85qUVwatsJoIUNIfCRsp7qO0iAmpGFZ4EELWSbC4= github.com/gomodule/redigo v1.8.0 h1:OXfLQ/k8XpYF8f8sZKd2Df4SDyzbLeC35OsBsB11rYg= github.com/gomodule/redigo v1.8.0/go.mod h1:P9dn9mFrCBvWhGE1wpxx6fgq7BAeLBk+UUUzlpkBYO0= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= @@ -370,6 +420,7 @@ github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck= github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8= github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17icRSOU623lUBU= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= @@ -392,8 +443,8 @@ github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38/go.mod h1:kpwsk12EmLe github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/subcommands v1.2.0/go.mod h1:ZjhPrFU+Olkh9WazFPsl27BQ4UPiG37m3yTrtFlrHVk= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= -github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.3.1 h1:KjJaJ9iWZ3jOFZIf1Lqf4laDRCasjl0BCmnEGxkdLb4= +github.com/google/uuid v1.3.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/enterprise-certificate-proxy v0.2.0 h1:y8Yozv7SZtlU//QXbezB6QkpuE6jMD2/gfzk4AftXjs= github.com/googleapis/enterprise-certificate-proxy v0.2.0/go.mod h1:8C0jb7/mgJe/9KK8Lm7X9ctZC2t60YyIpYEI16jx0Qg= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= @@ -410,6 +461,7 @@ github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI= github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= github.com/gorilla/securecookie v1.1.1 h1:miw7JPhV+b/lAHSXz4qd/nN9jRiAFV5FwjeKyCS8BvQ= github.com/gorilla/securecookie v1.1.1/go.mod h1:ra0sb63/xPlUeL+yeDciTfxMRAA+MP+HVt/4epWDjd4= +github.com/gorilla/websocket v1.4.1/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/gorilla/websocket v1.5.0 h1:PPwGk2jz7EePpoHN/+ClbZu8SPxiqlu12wZP/3sWmnc= github.com/gorilla/websocket v1.5.0/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= @@ -430,6 +482,7 @@ github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerX github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4= github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= +github.com/hashicorp/go-version v1.2.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= github.com/hashicorp/go-version v1.6.0 h1:feTTfFNnjP967rlCxM/I9g701jU+RN74YKx2mOkIeek= github.com/hashicorp/go-version v1.6.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90= @@ -442,22 +495,30 @@ github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0mNTz8vQ= github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I= github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc= -github.com/herumi/bls-eth-go-binary v0.0.0-20200522010937-01d282b5380b/go.mod h1:luAnRm3OsMQeokhGzpYmc0ZKwawY7o87PUEP11Z7r7U= github.com/herumi/bls-eth-go-binary v1.29.1 h1:XcNSHYTyNjEUVfWDCE2gtG5r95biTwd7MJUJF09LtSE= github.com/herumi/bls-eth-go-binary v1.29.1/go.mod h1:luAnRm3OsMQeokhGzpYmc0ZKwawY7o87PUEP11Z7r7U= github.com/holiman/billy v0.0.0-20230718173358-1c7e68d277a7 h1:3JQNjnMRil1yD0IfZKHF9GxxWKDJGj8I0IqOUol//sw= github.com/holiman/bloomfilter/v2 v2.0.3 h1:73e0e/V0tCydx14a0SCYS/EWCxgwLZ18CZcZKVu0fao= +github.com/holiman/bloomfilter/v2 v2.0.3/go.mod h1:zpoh+gs7qcpqrHr3dB55AMiJwo0iURXE7ZOP9L9hSkA= +github.com/holiman/uint256 v1.2.0/go.mod h1:y4ga/t+u+Xwd7CpDgZESaRcWy0I7XMlTMA25ApIH5Jw= github.com/holiman/uint256 v1.2.3 h1:K8UWO1HUJpRMXBxbmaY1Y8IAMZC/RsKB+ArEnnK4l5o= github.com/holiman/uint256 v1.2.3/go.mod h1:SC8Ryt4n+UBbPbIBKaG9zbbDlp4jOru9xFZmPzLUTxw= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/huin/goupnp v1.0.3 h1:N8No57ls+MnjlB+JPiCVSOyy/ot7MJTqlo7rn+NYSqQ= +github.com/hydrogen18/memlistener v0.0.0-20200120041712-dcc25e7acd91/go.mod h1:qEIFzExnS6016fRpRfxrExeVn2gbClQA99gQhnIcdhE= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/imdario/mergo v0.3.13 h1:lFzP57bqS/wsqKssCGmtLAb8A0wKjLGrve2q3PPVcBk= github.com/imdario/mergo v0.3.13/go.mod h1:4lJ1jqUDcsbIECGy0RUJAXNIhg+6ocWgb1ALK2O4oXg= +github.com/imkira/go-interpol v1.1.0/go.mod h1:z0h2/2T3XF8kyEPpRgJ3kmNv+C43p+I/CoI+jC3w2iA= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= -github.com/ipfs/go-cid v0.3.2 h1:OGgOd+JCFM+y1DjWPmVH+2/4POtpDzwcr7VgnB7mZXc= -github.com/ipfs/go-cid v0.3.2/go.mod h1:gQ8pKqT/sUxGY+tIwy1RPpAojYu7jAyCp5Tz1svoupw= +github.com/ipfs/go-cid v0.4.1 h1:A/T3qGvxi4kpKWWcPC/PgbvDA2bjVLO7n4UeVwnbs/s= +github.com/ipfs/go-cid v0.4.1/go.mod h1:uQHwDeX4c6CtyrFwdqyhpNcxVewur1M7l7fNU7LKwZk= +github.com/iris-contrib/blackfriday v2.0.0+incompatible/go.mod h1:UzZ2bDEoaSGPbkg6SAB4att1aAwTmVIx/5gCVqeyUdI= +github.com/iris-contrib/go.uuid v2.0.0+incompatible/go.mod h1:iz2lgM/1UnEf1kP0L/+fafWORmlnuysV2EMP8MW+qe0= +github.com/iris-contrib/jade v1.1.3/go.mod h1:H/geBymxJhShH5kecoiOCSssPX7QWYH7UaeZTSWddIk= +github.com/iris-contrib/pongo2 v0.0.1/go.mod h1:Ssh+00+3GAZqSQb30AvBRNxBx7rf0GqwkjqxNd0u65g= +github.com/iris-contrib/schema v0.0.1/go.mod h1:urYA3uvUNG1TIIjOSCzHr9/LmbQo8LrOcOqfqxa4hXw= github.com/jackc/chunkreader v1.0.0/go.mod h1:RT6O25fNZIuasFJRyZ4R/Y2BbhasbmZXF9QQ7T3kePo= github.com/jackc/chunkreader/v2 v2.0.0/go.mod h1:odVSm741yZoC3dpHEUXIqA9tQRhFrgOHwnPIn9lDKlk= github.com/jackc/chunkreader/v2 v2.0.1 h1:i+RDz65UE+mmpjTfyz0MoVTnzeYxroil2G82ki7MGG8= @@ -526,21 +587,32 @@ github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7V github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= github.com/jung-kurt/gofpdf v1.0.3-0.20190309125859-24315acbbda5 h1:PJr+ZMXIecYc1Ey2zucXdR73SMBtgjPgwa31099IMv0= github.com/jung-kurt/gofpdf v1.0.3-0.20190309125859-24315acbbda5/go.mod h1:7Id9E/uU8ce6rXgefFLlgrJj/GYY22cpxn+r32jIOes= +github.com/k0kubun/colorstring v0.0.0-20150214042306-9440f1994b88/go.mod h1:3w7q1U84EfirKl04SVQ/s7nPm1ZPhiXd34z40TNz36k= +github.com/kataras/golog v0.0.10/go.mod h1:yJ8YKCmyL+nWjERB90Qwn+bdyBZsaQwU3bTVFgkFIp8= github.com/kataras/i18n v0.0.5 h1:X9EQHxDhjpN0zh+Ry0PZvi0ODi9lf5mo4wiXWtOYhlY= github.com/kataras/i18n v0.0.5/go.mod h1:U0aKF7ANqGmFVs4WCexDTYGf8wg7Rb3mLJCmr/OuDoo= +github.com/kataras/iris/v12 v12.1.8/go.mod h1:LMYy4VlP67TQ3Zgriz8RE2h2kMZV2SgMYbq3UhfoFmE= +github.com/kataras/neffos v0.0.14/go.mod h1:8lqADm8PnbeFfL7CLXh1WHw53dG27MC3pgi2R1rmoTE= +github.com/kataras/pio v0.0.2/go.mod h1:hAoW0t9UmXi4R5Oyq5Z4irTbaTsOemSrDGUtaTl7Dro= +github.com/kataras/sitemap v0.0.5/go.mod h1:KY2eugMKiPwsJgx7+U103YZehfvNGOXURubcGyk0Bz8= github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 h1:Z9n2FFNUXsshfwJMBgNA0RU6/i7WVaAegv3PtuIHPMs= github.com/kelseyhightower/envconfig v1.4.0 h1:Im6hONhd3pLkfDFsbRgu68RDNkGF1r3dvMUtDTo2cv8= github.com/kelseyhightower/envconfig v1.4.0/go.mod h1:cccZRl6mQpaq41TPp5QxidR+Sa3axMbJDNb//FQX6Gg= +github.com/kilic/bls12-381 v0.1.0/go.mod h1:vDTTHJONJ6G+P2R74EhnyotQDTliQDnFEwhdmfzw1ig= github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= +github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/klauspost/compress v1.8.2/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= +github.com/klauspost/compress v1.9.7/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= github.com/klauspost/compress v1.11.7/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= github.com/klauspost/compress v1.16.0 h1:iULayQNOReoYUe+1qtKOqw9CwJv3aNQu8ivo7lw1HU4= github.com/klauspost/compress v1.16.0/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= +github.com/klauspost/cpuid v1.2.1/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= github.com/klauspost/cpuid/v2 v2.0.4/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/klauspost/cpuid/v2 v2.1.2/go.mod h1:RVVoqg1df56z8g3pUjL/3lE5UfnlrJX8tyFgg4nqhuY= -github.com/klauspost/cpuid/v2 v2.2.4 h1:acbojRNwl3o09bUq+yDCtZFc1aiwaAAxtcn8YkZXnvk= -github.com/klauspost/cpuid/v2 v2.2.4/go.mod h1:RVVoqg1df56z8g3pUjL/3lE5UfnlrJX8tyFgg4nqhuY= +github.com/klauspost/cpuid/v2 v2.2.5 h1:0E5MSMDEoAulmXNFquVs//DdoomxaoTY1kUhbc/qbZg= +github.com/klauspost/cpuid/v2 v2.2.5/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= github.com/klauspost/pgzip v1.2.5/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= @@ -548,13 +620,17 @@ github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxv github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/pty v1.1.8/go.mod h1:O1sed60cT9XZ5uDucP5qwvh+TE3NnUj51EiZO/lmSfw= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= +github.com/labstack/echo/v4 v4.5.0/go.mod h1:czIriw4a0C1dFun+ObrXp7ok03xON0N1awStJ6ArI7Y= +github.com/labstack/gommon v0.3.0/go.mod h1:MULnywXg0yavhxWKc+lOruYdAhDwPK9wf0OL7NoOu+k= github.com/leanovate/gopter v0.2.9 h1:fQjYxZaynp97ozCzfOyOuAGOU4aU/z37zf/tOujFk7c= github.com/leodido/go-urn v1.1.0/go.mod h1:+cyI34gQWZcE1eQU7NVgKkkzdXDQHr1dBMtdAPozLkw= github.com/leodido/go-urn v1.2.0/go.mod h1:+8+nEpDfqqsY+g338gtMEUOtuK+4dEMhiQEgxpxOKII= @@ -565,6 +641,7 @@ github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/lib/pq v1.10.2/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= github.com/lib/pq v1.10.7 h1:p7ZhMD+KsSRozJr34udlUrhboJwWAgCg34+/ZZNvZZw= github.com/lib/pq v1.10.7/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= +github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/mailgun/mailgun-go/v4 v4.1.3 h1:KLa5EZaOMMeyvY/lfAhWxv9ealB3mtUsMz0O9XmTtP0= github.com/mailgun/mailgun-go/v4 v4.1.3/go.mod h1:R9kHUQBptF4iSEjhriCQizplCDwrnDShy8w/iPiOfaM= @@ -576,32 +653,42 @@ github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0 github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcnceauSikq3lYCQ= +github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= github.com/mattn/go-colorable v0.1.6/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= github.com/mattn/go-colorable v0.1.8/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= github.com/mattn/go-colorable v0.1.9/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= +github.com/mattn/go-colorable v0.1.11/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4= github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.7/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= +github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.9/go.mod h1:YNRxwqDuOph6SZLI9vUUz6OYw3QyUt7WiY2yME+cCiQ= github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= github.com/mattn/go-isatty v0.0.17 h1:BTarxUcIeDqL27Mc+vyvdWYSL28zpIhv3RoTdsLMPng= github.com/mattn/go-isatty v0.0.17/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= +github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= github.com/mattn/go-runewidth v0.0.14 h1:+xnbZSEeDbOIg5/mE6JF0w6n9duR1l3/WmbinWVwUuU= -github.com/mattn/go-sqlite3 v1.9.0 h1:pDRiWfl+++eC2FEFRy6jXmQlvp4Yh3z1MJKg4UeYM/4= +github.com/mattn/go-runewidth v0.0.14/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= github.com/mattn/go-sqlite3 v1.9.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= +github.com/mattn/go-sqlite3 v1.11.0 h1:LDdKkqtYlom37fkvqs8rMPFKAMe8+SgjbwZ6ex1/A/Q= +github.com/mattn/go-sqlite3 v1.11.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= +github.com/mattn/goveralls v0.0.2/go.mod h1:8d1ZMHsd7fW6IRPKQh46F2WRpyib5/X4FOpevwGNQEw= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo= github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= +github.com/mediocregopher/radix/v3 v3.4.2/go.mod h1:8FL3F6UQRXHXIBSPUs5h0RybMF8i4n7wVopoX3x7Bv8= +github.com/microcosm-cc/bluemonday v1.0.2/go.mod h1:iVP4YcDBq+n/5fb23BhYFvIMq/leAFZyRl6bYmGDlGc= github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= github.com/minio/highwayhash v1.0.2 h1:Aak5U0nElisjDCfPSG79Tgzkn2gl66NxOMspRrKnA/g= github.com/minio/highwayhash v1.0.2/go.mod h1:BQskDq+xkJ12lmlUUi7U0M5Swg3EWR+dLTk+kldvVxY= github.com/minio/sha256-simd v0.1.0/go.mod h1:2FMWW+8GMoPweT6+pI63m9YE3Lmw4J71hV56Chs1E/U= -github.com/minio/sha256-simd v1.0.0 h1:v1ta+49hkWZyvaKwrQB8elexRqm6Y0aMLjCNsrYxo6g= github.com/minio/sha256-simd v1.0.0/go.mod h1:OuYzVNI5vcoYIAmbIvHPl3N3jUzVedXbKy5RFepssQM= +github.com/minio/sha256-simd v1.0.1 h1:6kaan5IFmwTNynnKKpDHe6FWHohJOHhCPchzK49dzMM= +github.com/minio/sha256-simd v1.0.1/go.mod h1:Pz6AKMiUdngCLpeTL/RJY1M9rUuPMYujV5xJjtbRSN8= github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= @@ -626,6 +713,7 @@ github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3Rllmb github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 h1:RWengNIwukTxcDr9M+97sNutRR1RKhG96O6jWumTTnw= github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826/go.mod h1:TaXosZuwdSHYgviHp1DAtfrULt5eUgsSMsZf+YrPgl8= +github.com/moul/http2curl v1.0.0/go.mod h1:8UbvGypXm98wA/IqH45anm5Y2Z6ep6O31QGOAZ3H0fQ= github.com/mr-tron/base58 v1.2.0 h1:T/HDJBh4ZCPbU39/+c3rRvE0uKBQlU27+QI8LJ4t64o= github.com/mr-tron/base58 v1.2.0/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjWI2mNwc= github.com/mssola/user_agent v0.5.2 h1:CZkTUahjL1+OcZ5zv3kZr8QiJ8jy2H08vZIEkBeRbxo= @@ -634,16 +722,20 @@ github.com/multiformats/go-base32 v0.1.0 h1:pVx9xoSPqEIQG8o+UbAe7DNi51oej1NtK+aG github.com/multiformats/go-base32 v0.1.0/go.mod h1:Kj3tFY6zNr+ABYMqeUNeGvkIC/UYgtWibDcT0rExnbI= github.com/multiformats/go-base36 v0.2.0 h1:lFsAbNOGeKtuKozrtBsAkSVhv1p9D0/qedU9rQyccr0= github.com/multiformats/go-base36 v0.2.0/go.mod h1:qvnKE++v+2MWCfePClUEjE78Z7P2a1UV0xHgWc0hkp4= -github.com/multiformats/go-multibase v0.1.1 h1:3ASCDsuLX8+j4kx58qnJ4YFq/JWTJpCyDW27ztsVTOI= -github.com/multiformats/go-multibase v0.1.1/go.mod h1:ZEjHE+IsUrgp5mhlEAYjMtZwK1k4haNkcaPg9aoe1a8= -github.com/multiformats/go-multihash v0.2.1 h1:aem8ZT0VA2nCHHk7bPJ1BjUbHNciqZC/d16Vve9l108= -github.com/multiformats/go-multihash v0.2.1/go.mod h1:WxoMcYG85AZVQUyRyo9s4wULvW5qrI9vb2Lt6evduFc= +github.com/multiformats/go-multibase v0.2.0 h1:isdYCVLvksgWlMW9OZRYJEa9pZETFivncJHmHnnd87g= +github.com/multiformats/go-multibase v0.2.0/go.mod h1:bFBZX4lKCA/2lyOFSAoKH5SS6oPyjtnzK/XTFDPkNuk= +github.com/multiformats/go-multihash v0.2.3 h1:7Lyc8XfX/IY2jWb/gI7JP+o7JEq9hOa7BFvVU9RSh+U= +github.com/multiformats/go-multihash v0.2.3/go.mod h1:dXgKXCXjBzdscBLk9JkjINiEsCKRVch90MdaGiKsvSM= github.com/multiformats/go-varint v0.0.7 h1:sWSGR+f/eu5ABZA2ZpYKBILXTTs9JWpdEM/nEGOHFS8= github.com/multiformats/go-varint v0.0.7/go.mod h1:r8PUYw/fD/SjBCiKOoDlGF6QawOELpZAu9eioSos/OU= github.com/mvdan/xurls v1.1.0 h1:OpuDelGQ1R1ueQ6sSryzi6P+1RtBpfQHM8fJwlE45ww= github.com/mvdan/xurls v1.1.0/go.mod h1:tQlNn3BED8bE/15hnSL2HLkDeLWpNPAwtw7wkEq44oU= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= +github.com/nats-io/jwt v0.3.0/go.mod h1:fRYCDE99xlTsqUzISS1Bi75UBJ6ljOJQOAAu5VglpSg= +github.com/nats-io/nats.go v1.9.1/go.mod h1:ZjDU1L/7fJ09jvUSRVBR2e7+RnLiiIQyqyzEE/Zbp4w= +github.com/nats-io/nkeys v0.1.0/go.mod h1:xpnFELMwJABBLVhffcfd1MZx6VsNRFpEugbxziKVo7w= +github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= github.com/nishanths/predeclared v0.0.0-20200524104333-86fad755b4d3/go.mod h1:nt3d53pc1VYcphSCIaYAJtnPYnr3Zyn8fMq2wvPGPso= github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= @@ -651,7 +743,9 @@ github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= github.com/olekukonko/tablewriter v0.0.5 h1:P2Ga83D34wi1o9J6Wh1mRuqd4mF/x/lgBS7N7AbDhec= +github.com/olekukonko/tablewriter v0.0.5/go.mod h1:hPp6KlRPjbx+hW8ykQs1w3UBbZlj6HuIJcUGPhkA7kY= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.10.3/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vvnwo0= github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE= @@ -670,6 +764,9 @@ github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58/go.mod h1:DXv8WO4yhM github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/phyber/negroni-gzip v0.0.0-20180113114010-ef6356a5d029 h1:d6HcSW4ZoNlUWrPyZtBwIu8yv4WAWIU3R/jorwVkFtQ= github.com/phyber/negroni-gzip v0.0.0-20180113114010-ef6356a5d029/go.mod h1:94RTq2fypdZCze25ZEZSjtbAQRT3cL/8EuRUqAZC/+w= +github.com/pingcap/errors v0.11.4 h1:lFuQV/oaUMGcD2tqt+01ROSmJs75VG1ToEOkZIZ4nE4= +github.com/pingcap/errors v0.11.4/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8= +github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= @@ -713,12 +810,13 @@ github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1 github.com/prometheus/procfs v0.9.0 h1:wzCHvIvM5SxWqYvwgVL7yJY8Lz3PKn49KQtpgMYJfhI= github.com/prometheus/procfs v0.9.0/go.mod h1:+pB4zwohETzFnmlpe6yd2lSc+0/46IYZRB/chUwxUZY= github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= +github.com/protolambda/bls12-381-util v0.0.0-20210720105258-a772f2aac13e/go.mod h1:MPZvj2Pr0N8/dXyTPS5REeg2sdLG7t8DRzC1rLv925w= github.com/protolambda/messagediff v1.4.0/go.mod h1:LboJp0EwIbJsePYpzh5Op/9G1/4mIztMRYzzwR0dR2M= -github.com/protolambda/zrnt v0.12.4 h1:BJE8HSYx9wJnOOBr/QdYCE0fxUOfJpSKbSCcXGprm28= -github.com/protolambda/zrnt v0.12.4/go.mod h1:7G3u3+BgUjmQw75fQ3OYOJucujRrspBmshCx3OdRutc= +github.com/protolambda/zrnt v0.30.0 h1:pHEn69ZgaDFGpLGGYG1oD7DvYI7RDirbMBPfbC+8p4g= +github.com/protolambda/zrnt v0.30.0/go.mod h1:qcdX9CXFeVNCQK/q0nswpzhd+31RHMk2Ax/2lMsJ4Jw= github.com/protolambda/zssz v0.1.5 h1:7fjJjissZIIaa2QcvmhS/pZISMX21zVITt49sW1ouek= github.com/protolambda/zssz v0.1.5/go.mod h1:a4iwOX5FE7/JkKA+J/PH0Mjo9oXftN6P8NZyL28gpag= -github.com/protolambda/ztyp v0.1.0/go.mod h1:u9yT4ioIokwlHrOfiZ52ezHfKdza3phxhTJix01vGgU= +github.com/protolambda/ztyp v0.2.2/go.mod h1:9bYgKGqg3wJqT9ac1gI2hnVb0STQq7p/1lapqrqY1dU= github.com/prysmaticlabs/fastssz v0.0.0-20221107182844-78142813af44 h1:c3p3UzV4vFA7xaCDphnDWOjpxcadrQ26l5b+ypsvyxo= github.com/prysmaticlabs/fastssz v0.0.0-20221107182844-78142813af44/go.mod h1:MA5zShstUwCQaE9faGHgCGvEWUbG87p4SAXINhmCkvg= github.com/prysmaticlabs/go-bitfield v0.0.0-20210809151128-385d8c5e3fb7 h1:0tVE4tdWQK9ZpYygoV7+vS6QkDvQVySboMVEIxBJmXw= @@ -734,7 +832,9 @@ github.com/prysmaticlabs/protoc-gen-go-cast v0.0.0-20211014160335-757fae4f38c6 h github.com/r3labs/sse/v2 v2.7.4 h1:pvCMswPDlXd/ZUFx1dry0LbXJNHXwWPulLcUGYwClc0= github.com/r3labs/sse/v2 v2.7.4/go.mod h1:hUrYMKfu9WquG9MyI0r6TKiNH+6Sw/QPKm2YbNbU5g8= github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE= +github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= github.com/rivo/uniseg v0.4.3 h1:utMvzDsuh3suAEnhH0RdHmoPbU648o6CvXxTx4SBMOw= +github.com/rivo/uniseg v0.4.3/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= github.com/rocket-pool/go-merkletree v1.0.1-0.20220406020931-c262d9b976dd h1:p9KuetSKB9nte9I/MkkiM3pwKFVQgqxxPTQ0y56Ff6s= github.com/rocket-pool/go-merkletree v1.0.1-0.20220406020931-c262d9b976dd/go.mod h1:UE9fof8P7iESVtLn1K9CTSkNRYVFHZHlf96RKbU33kA= github.com/rocket-pool/rocketpool-go v1.10.1-0.20230228020137-d5a680907dff h1:rOZevts77yp6t7J9xPRxhM1s4xyp4vbfqQJlFIWdDGE= @@ -744,7 +844,10 @@ github.com/rocket-pool/smartnode v1.9.3/go.mod h1:uz3BxUtmG+R7BdjAQ3ogBuxPdpZvvt github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= +github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= +github.com/rogpeppe/go-internal v1.8.1/go.mod h1:JeRgkft04UBgHMgCIwADu4Pn6Mtm5d4nPKWu0nJ5d+o= github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8= +github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= github.com/rs/cors v1.8.0 h1:P2KMzcFwrPoSjkF1WLRPsp3UMLyql8L4v9hQpVeK5so= github.com/rs/cors v1.8.0/go.mod h1:EBwu+T5AvHOcXwvZIkQFjUN6s8Czyqw12GL/Y0tUyRM= github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ= @@ -753,12 +856,16 @@ github.com/rs/zerolog v1.13.0/go.mod h1:YbFCdg8HfsridGWAh22vktObvhZbQsZXe4/zB0OK github.com/rs/zerolog v1.15.0/go.mod h1:xYTKnLHcpfU2225ny5qZjxnj9NvkumZYjJHlAThCjNc= github.com/rs/zerolog v1.26.1 h1:/ihwxqH+4z8UxyI70wM1z9yCvkWcfz/a3mj48k/Zngc= github.com/rs/zerolog v1.26.1/go.mod h1:/wSSJWX7lVrsOwlbyTRSOJvqRlc+WjWlfes+CiJ+tmc= +github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= +github.com/ryanuber/columnize v2.1.0+incompatible/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= +github.com/schollz/closestmatch v2.1.0+incompatible/go.mod h1:RtP1ddjLong6gTkbtmuhtR2uUrrJOpYzYRvbcPAid+g= github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= +github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= github.com/shirou/gopsutil v3.21.11+incompatible h1:+1+c1VGhc88SSonWP6foOcLhvnKlUeu/erjjvaPEYiI= github.com/shirou/gopsutil v3.21.11+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA= github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24/go.mod h1:M+9NzErvs504Cn4c5DxATwIqPbtswREoFCre64PpcG4= @@ -782,10 +889,12 @@ github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0b github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= +github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU= github.com/spf13/cobra v1.0.1-0.20201006035406-b97b5ead31f7/go.mod h1:yk5b0mALVusDL5fMM6Rd1wgnoO5jUPhwsQ6LQAJTidQ= github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s= github.com/spf13/viper v1.7.0/go.mod h1:8WkrPz2fc9jxqZNCJI/76HCieCp4Q8HaLFoCha5qpdg= github.com/status-im/keycard-go v0.2.0 h1:QDLFswOQu1r5jsycloeQh3bVU8n/NatHHaZobtDnDzA= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= @@ -803,7 +912,7 @@ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.7.2/go.mod h1:R6va5+xMeoiuVRoj+gSkQ7d3FALtqAAGI1FQKckRals= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= -github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8= +github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= github.com/stripe/stripe-go/v72 v72.50.0 h1:oy+EsSKMrFS3zzayb8Ic+2LZ04Ux0vJ4990/7psaYsc= github.com/stripe/stripe-go/v72 v72.50.0/go.mod h1:QwqJQtduHubZht9mek5sds9CtQcKFdsykV9ZepRWwo0= github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= @@ -819,15 +928,17 @@ github.com/syndtr/goleveldb v1.0.1-0.20220721030215-126854af5e6d h1:vfofYNRScrDd github.com/syndtr/goleveldb v1.0.1-0.20220721030215-126854af5e6d/go.mod h1:RRCYJbIwD5jmqPI9XoAFR0OcDxqUctll6zUj/+B4S48= github.com/thomaso-mirodin/intmath v0.0.0-20160323211736-5dc6d854e46e h1:cR8/SYRgyQCt5cNCMniB/ZScMkhI9nk8U5C7SbISXjo= github.com/thomaso-mirodin/intmath v0.0.0-20160323211736-5dc6d854e46e/go.mod h1:Tu4lItkATkonrYuvtVjG0/rhy15qrNGNTjPdaphtZ/8= -github.com/tklauser/go-sysconf v0.3.11 h1:89WgdJhk5SNwJfu+GKyYveZ4IaJ7xAkecBo+KdJV0CM= -github.com/tklauser/go-sysconf v0.3.11/go.mod h1:GqXfhXY3kiPa0nAXPDIQIWzJbMCB7AmcWpGR8lSZfqI= -github.com/tklauser/numcpus v0.6.0 h1:kebhY2Qt+3U6RNK7UqpYNA+tJ23IBEGKkB7JQBfDYms= -github.com/tklauser/numcpus v0.6.0/go.mod h1:FEZLMke0lhOUG6w2JadTzp0a+Nl8PF/GFkQ5UVIcaL4= +github.com/tklauser/go-sysconf v0.3.12 h1:0QaGUFOdQaIVdPgfITYzaTegZvdCjmYO52cSFAEVmqU= +github.com/tklauser/go-sysconf v0.3.12/go.mod h1:Ho14jnntGE1fpdOqQEEaiKRpvIavV0hSfmBq8nJbHYI= +github.com/tklauser/numcpus v0.6.1 h1:ng9scYS7az0Bk4OZLvrNXNSAO2Pxr1XXRAPyjhIx+Fk= +github.com/tklauser/numcpus v0.6.1/go.mod h1:1XfjsgE2zo8GVw7POkMbHENHzVg3GzmoZ9fESEdAacY= github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/twitchtv/twirp v7.1.0+incompatible/go.mod h1:RRJoFSAmTEh2weEqWtpPE3vFK5YBhA6bqp2l1kfCC5A= github.com/tyler-smith/go-bip39 v1.1.0 h1:5eUemwrMargf3BSLRRCalXT93Ns6pQJIjYQN2nyfOP8= github.com/uber/jaeger-client-go v2.25.0+incompatible h1:IxcNZ7WRY1Y3G4poYlx24szfsn/3LvK9QHCq9oQw8+U= +github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc= github.com/ugorji/go v1.1.7/go.mod h1:kZn38zHttfInRq0xu/PH0az30d+z6vm202qpg1oXVMw= +github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= github.com/ugorji/go/codec v1.1.7/go.mod h1:Ax+UKWsSmolVDwsd+7N3ZtXu+yMGCf907BLYF3GoBXY= github.com/umbracle/gohashtree v0.0.2-alpha.0.20230207094856-5b775a815c10 h1:CQh33pStIp/E30b7TxDlXfM0145bn2e8boI30IxAhTg= github.com/urfave/cli v1.22.12 h1:igJgVw1JdKH+trcLWLeLwZjU9fEfPesQ+9/e4MQ44S8= @@ -835,23 +946,37 @@ github.com/urfave/cli v1.22.12/go.mod h1:sSBEIC79qR6OvcmsD4U3KABeOTxDqQtdDnaFuUN github.com/urfave/cli/v2 v2.24.1 h1:/QYYr7g0EhwXEML8jO+8OYt5trPnLHS0p3mrgExJ5NU= github.com/urfave/negroni v1.0.0 h1:kIimOitoypq34K7TG7DUaJ9kq/N4Ofuwi1sjz0KipXc= github.com/urfave/negroni v1.0.0/go.mod h1:Meg73S6kFm/4PpbYdq35yYWoCZ9mS/YSx+lKnmiohz4= +github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= +github.com/valyala/fasthttp v1.6.0/go.mod h1:FstJa9V+Pj9vQ7OJie2qMHdwemEDaDiSdBnvPM1Su9w= +github.com/valyala/fasttemplate v1.0.1/go.mod h1:UQGH1tvbgY+Nz5t2n7tXsz52dQxojPUpymEIMZ47gx8= +github.com/valyala/fasttemplate v1.2.1/go.mod h1:KHLXt3tVN2HBp8eijSv/kGJopbvo7S+qRAEEKiv+SiQ= +github.com/valyala/tcplisten v0.0.0-20161114210144-ceec8f93295a/go.mod h1:v3UYOV9WzVtRmSR+PDvWpU/qWl4Wa5LApYYX4ZtKbio= github.com/wealdtech/go-bytesutil v1.2.1 h1:TjuRzcG5KaPwaR5JB7L/OgJqMQWvlrblA1n0GfcXFSY= github.com/wealdtech/go-bytesutil v1.2.1/go.mod h1:RhUDUGT1F4UP4ydqbYp2MWJbAel3M+mKd057Pad7oag= -github.com/wealdtech/go-ens/v3 v3.5.5 h1:/jq3CDItK0AsFnZtiFJK44JthkAMD5YE3WAJOh4i7lc= -github.com/wealdtech/go-ens/v3 v3.5.5/go.mod h1:w0EDKIm0dIQnqEKls6ORat/or+AVfPEdEXVfN71EeEE= +github.com/wealdtech/go-ens/v3 v3.6.0 h1:EAByZlHRQ3vxqzzwNi0GvEq1AjVozfWO4DMldHcoVg8= +github.com/wealdtech/go-ens/v3 v3.6.0/go.mod h1:hcmMr9qPoEgVSEXU2Bwzrn/9NczTWZ1rE53jIlqUpzw= github.com/wealdtech/go-eth2-types/v2 v2.8.1 h1:y2N3xSIZ3tVqsnvj4AgPkh48U5sM612vhZwlK3k+3lM= github.com/wealdtech/go-eth2-types/v2 v2.8.1/go.mod h1:3TJShI4oBzG8pCZsfe3NZAq8QAmXrC2rd45q7Vn/XB8= github.com/wealdtech/go-eth2-util v1.8.1 h1:nb50hygsNoql94akg7GN6im/weg8ZZgJWHgiyrj8qiU= github.com/wealdtech/go-eth2-util v1.8.1/go.mod h1:vv+8jVgYRXEGty/VLPNn1RYlbQNYmTht3VR6nfh0z4E= github.com/wealdtech/go-multicodec v1.4.0 h1:iq5PgxwssxnXGGPTIK1srvt6U5bJwIp7k6kBrudIWxg= github.com/wealdtech/go-multicodec v1.4.0/go.mod h1:aedGMaTeYkIqi/KCPre1ho5rTb3hGpu/snBOS3GQLw4= -github.com/wealdtech/go-string2eth v1.1.0 h1:USJQmysUrBYYmZs7d45pMb90hRSyEwizP7lZaOZLDAw= +github.com/wealdtech/go-string2eth v1.2.1 h1:u9sofvGFkp+uvTg4Nvsvy5xBaiw8AibGLLngfC4F76g= +github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= +github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ= +github.com/xeipuuv/gojsonschema v1.2.0/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQluxsYJ78Id3Y= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= +github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 h1:bAn7/zixMGCfxrRTfdpNzjtPYqr8smhKouy9mxVdGPU= +github.com/yalp/jsonpath v0.0.0-20180802001716-5cc68e5049a0/go.mod h1:/LWChgwKmvncFJFHJ7Gvn9wZArjbV5/FppcK2fKk/tI= +github.com/yudai/gojsondiff v1.0.0/go.mod h1:AY32+k2cwILAkW1fbgxQ5mUmMiZFgLIV+FBNExI05xg= +github.com/yudai/golcs v0.0.0-20170316035057-ecda9a501e82/go.mod h1:lgjkn3NuSvDfVJdfcVVdX+jpBxNmX4rDAzaS45IcYoM= +github.com/yudai/pp v2.0.1+incompatible/go.mod h1:PuxR/8QJ7cyCkFp/aUDS+JY727OFEZkTdatxwunjIkc= github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/yuin/goldmark v1.4.0/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= github.com/yusufpapurcu/wmi v1.2.2 h1:KBNDSne4vP5mbSWnJbO+51IMOXJB67QiYCSBrubbPRg= @@ -886,21 +1011,25 @@ go.uber.org/zap v1.13.0/go.mod h1:zwrFLgMcdUuIBviXEYEH1YKNaOBnKXsx2IPda5bBwHM= go.uber.org/zap v1.16.0/go.mod h1:MA8QOfq0BHJwdXa996Y4dYkAqRKB8/1K1QMMZVaNZjQ= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190411191339-88737f569e3a/go.mod h1:WFFai1msRO1wXaEeE5yQxYXgSfI8pQAWXbQop6sCtWE= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20191227163750-53104e6ec876/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20201203163018-be400aefbc4c/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= +golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= golang.org/x/crypto v0.0.0-20210616213533-5ff15b29337e/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20211215165025-cf75a172585e/go.mod h1:P+XmwS30IXTQdn5tA2iutPOUgjI07+tq3H3K9MVA1s8= golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58= -golang.org/x/crypto v0.12.0 h1:tFM/ta59kqch6LlvYnPa0yx5a83cL2nHflFhYKvv9Yk= -golang.org/x/crypto v0.12.0/go.mod h1:NF0Gs7EO5K4qLn+Ylc+fih8BSTeIjAP05siRnAh98yw= +golang.org/x/crypto v0.13.0 h1:mvySKfSWJ+UKUii46M40LOvyWfN0s2U+46/jDd0e6Ck= +golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -911,8 +1040,8 @@ golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u0 golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= -golang.org/x/exp v0.0.0-20230810033253-352e893a4cad h1:g0bG7Z4uG+OgH2QDODnjp6ggkk1bJDsINcuWmJN1iJU= -golang.org/x/exp v0.0.0-20230810033253-352e893a4cad/go.mod h1:FXUEEKJgO7OQYeo8N01OfiKP8RXMtf6e8aTskBGqWdc= +golang.org/x/exp v0.0.0-20230905200255-921286631fa9 h1:GoHiUyI/Tp2nVkLI2mCxVkOjsbSXD66ic0XW0js0R9g= +golang.org/x/exp v0.0.0-20230905200255-921286631fa9/go.mod h1:S2oDrQGGwySpoQPVqRShND87VCbxmc6bL1Yd2oYrm6k= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= @@ -926,6 +1055,7 @@ golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRu golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/lint v0.0.0-20210508222113-6edffad5e616/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= @@ -937,8 +1067,7 @@ golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= -golang.org/x/mod v0.11.0 h1:bUO06HqtnRcc/7l71XBe4WcqTZ+3AH1J59zWDDwLKgU= -golang.org/x/mod v0.11.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.12.0 h1:rmsUpXtvNzj340zd98LZ4KntptpfRHwpFOHG188oHXc= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -949,6 +1078,7 @@ golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73r golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190327091125-710a502c58a2/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= @@ -958,6 +1088,7 @@ golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20191116160921-f9c825593386/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= @@ -982,16 +1113,18 @@ golang.org/x/net v0.0.0-20201209123823-ac852fbbde11/go.mod h1:m0MpNAwzfU5UDzcl9v golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4/go.mod h1:RBQZq4jEuRlivfhVLdyRGr576XBO4/greRjx4P4O3yc= +golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk= golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20211008194852-3b03d305991f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220607020251-c690dde0001d/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= -golang.org/x/net v0.10.0 h1:X2//UzNDwYmtCLn7To6G58Wr6f5ahEAQgKNzv9Y951M= -golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= +golang.org/x/net v0.15.0 h1:ugBLEUaxABaB5AJqW9enI0ACdci2RUd4eP51NTBvuJ8= +golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -1025,6 +1158,7 @@ golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20181026203630-95b1ffbd15a5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190130150945-aca44879d564/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -1036,6 +1170,7 @@ golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -1066,6 +1201,7 @@ golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201101102859-da207088b7d1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201214095126-aec9a390925b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -1074,13 +1210,20 @@ golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210315160823-c6e025ad8005/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210324051608-47abb6519492/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220209214540-3681064d5158/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220406163625-3f8b81556e12/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220412211240-33da011f77ad/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -1090,15 +1233,18 @@ golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20221010170243-090e33056c14/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.11.0 h1:eG7RXZHdqOJ1i+0lgLgCpSXAp6M3LYlAo6osgSi0xOM= +golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.12.0 h1:CM0HF96J0hcLAwsHPJZjfdNzs0gftsLfgKt57wWHJ0o= +golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= -golang.org/x/term v0.11.0 h1:F9tnn/DA/Im8nCwm+fX+1/eBwi4qFjRT++MhtVC4ZX0= +golang.org/x/term v0.12.0 h1:/ZfYdc3zq+q02Rv9vGqTeSItdzZTSNDmfTi0mBAuidU= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -1109,19 +1255,22 @@ golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= -golang.org/x/text v0.12.0 h1:k+n5B8goJNdU7hSvEtMUz3d1Q6D/XW4COJSJR6fN0mc= -golang.org/x/text v0.12.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= +golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k= +golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20201208040808-7e3f01d25324/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4= golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20181221001348-537d06c36207/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190327201419-c70d86f8b7cf/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190425163242-31fd60d6bfdc/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= @@ -1161,6 +1310,7 @@ golang.org/x/tools v0.0.0-20200512131952-2bc93b1c0c88/go.mod h1:EkVYQZoAsY45+roY golang.org/x/tools v0.0.0-20200515010526-7d3b6ebf133d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200522201501-cb1345f3a375/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200717024301-6ddee64345a6/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= @@ -1171,11 +1321,13 @@ golang.org/x/tools v0.0.0-20201201161351-ac6f37ff4c2a/go.mod h1:emZCQorbCU4vsT4f golang.org/x/tools v0.0.0-20201202200335-bef1c476418a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20201208233053-a543418bbed2/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= +golang.org/x/tools v0.1.3/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.7/go.mod h1:LGqMHiF4EqQNHR1JncWGqT5BVaXmza+X+BDGol+dOxo= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= -golang.org/x/tools v0.9.1 h1:8WMNJAz3zrtPmnYC7ISf5dEn3MT0gY7jBJfw27yrrLo= -golang.org/x/tools v0.9.1/go.mod h1:owI94Op576fPu3cIGQeHs3joujW/2Oc6MtlxbF5dfNc= +golang.org/x/tools v0.13.0 h1:Iey4qkscZuv0VvIt8E0neZjtPVQFSc870HQ448QgEmQ= +golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58= golang.org/x/xerrors v0.0.0-20190410155217-1f06c39b4373/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20190513163551-3ee3066db522/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -1215,6 +1367,7 @@ google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCID google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c= google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/genproto v0.0.0-20180518175338-11a468237815/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= @@ -1253,8 +1406,10 @@ google.golang.org/genproto v0.0.0-20201210142538-e3217bee35cc/go.mod h1:FWY/as6D google.golang.org/genproto v0.0.0-20201211151036-40ec1c210f7a/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210207032614-bba0dbe2a9ea/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210426193834-eac7f76ac494/go.mod h1:P3QM42oQyzQSnHPnZ/vqoCdDmzH28fzWByN9asMeM8A= +google.golang.org/genproto v0.0.0-20210624195500-8bfb893ecb84/go.mod h1:SzzZ/N+nwJDaO1kznhnlzqS8ocJICar6hYhVyhi++24= google.golang.org/genproto v0.0.0-20221118155620-16455021b5e6 h1:a2S6M0+660BgMNl++4JPlcAO/CjkqYItDEZwkoDQK7c= google.golang.org/genproto v0.0.0-20221118155620-16455021b5e6/go.mod h1:rZS5c/ZVYMaOGBfO68GWtjOw/eLaZM1X6iVtgjZ+EWg= +google.golang.org/grpc v1.12.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= @@ -1275,6 +1430,7 @@ google.golang.org/grpc v1.35.0-dev.0.20201218190559-666aea1fb34c/go.mod h1:qjiiY google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= google.golang.org/grpc v1.36.1/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= google.golang.org/grpc v1.37.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= +google.golang.org/grpc v1.38.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= google.golang.org/grpc v1.52.3 h1:pf7sOysg4LdgBqduXveGKrcEwbStiK2rtfghdzlUYDQ= google.golang.org/grpc v1.52.3/go.mod h1:pu6fVzoFb+NBYNAvQL08ic+lvB2IojljRYuun5vorUY= google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.0.0/go.mod h1:6Kw0yEErY5E/yWrBtf03jp27GLLJujG4z/JK95pnjjw= @@ -1306,14 +1462,19 @@ gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntN gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/go-playground/assert.v1 v1.2.1/go.mod h1:9RXL0bg/zibRAgZUYszZSwO/z8Y/a8bDuhia5mkpMnE= +gopkg.in/go-playground/validator.v8 v8.18.2/go.mod h1:RX2a/7Ha8BgOhfk7j780h4/u/RRjR0eouCJSH80/M2Y= gopkg.in/go-playground/validator.v9 v9.29.1/go.mod h1:+c9/zcJMFNgbLvly1L1V+PpxWdVbfP1avr/N00E2vyQ= gopkg.in/inconshreveable/log15.v2 v2.0.0-20180818164646-67afb5ed74ec/go.mod h1:aPpfJ7XW+gOuirDoZ8gHhLh3kZ1B08FtV2bbmy7Jv3s= gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= +gopkg.in/ini.v1 v1.51.1/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/ini.v1 v1.61.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= +gopkg.in/mgo.v2 v2.0.0-20180705113604-9856a29383ce/go.mod h1:yeKp02qBN3iKW1OzL3MGk2IdtZzaj7SFntXj72NppTA= gopkg.in/natefinch/lumberjack.v2 v2.0.0 h1:1Lc07Kr7qY4U2YPouBjpCLxpiyxIVoxqXgkXLknAOE8= +gopkg.in/natefinch/npipe.v2 v2.0.0-20160621034901-c1b8fa8bdcce h1:+JknDZhAj8YMt7GC73Ei8pv4MzjDUNPHgQWJdtMAaDU= +gopkg.in/natefinch/npipe.v2 v2.0.0-20160621034901-c1b8fa8bdcce/go.mod h1:5AcXVHNjg+BDxry382+8OKon8SEWiKktQR07RKPsv1c= gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= @@ -1327,6 +1488,7 @@ gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= +gopkg.in/yaml.v3 v3.0.0-20191120175047-4206685974f2/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= @@ -1348,8 +1510,8 @@ k8s.io/client-go v0.25.0/go.mod h1:lxykvypVfKilxhTklov0wz1FoaUZ8X4EwbhS6rpRfN8= k8s.io/klog v1.0.0 h1:Pt+yjF5aB1xDSVbau4VsWe+dQNzA0qv1LlXdC2dF6Q8= k8s.io/klog/v2 v2.70.1 h1:7aaoSdahviPmR+XkS7FyxlkkXs6tHISSG03RxleQAVQ= k8s.io/utils v0.0.0-20220728103510-ee6ede2d64ed h1:jAne/RjBTyawwAy0utX5eqigAwz/lQhTmy+Hr/Cpue4= -lukechampine.com/blake3 v1.1.7 h1:GgRMhmdsuK8+ii6UZFDL8Nb+VyMwadAgcJyfYHxG6n0= -lukechampine.com/blake3 v1.1.7/go.mod h1:tkKEOtDkNtklkXtLNEOGNq5tcV90tJiA1vAA12R78LA= +lukechampine.com/blake3 v1.2.1 h1:YuqqRuaqsGV71BV/nm9xlI0MKUv4QC54jQnBChWbGnI= +lukechampine.com/blake3 v1.2.1/go.mod h1:0OFRp7fBtAylGVCO40o87sbupkyIGgbpv1+M1k1LM6k= lukechampine.com/uint128 v1.2.0 h1:mBi/5l91vocEN8otkC5bDLhi2KdCticRiwbdB0O+rjI= modernc.org/cc/v3 v3.40.0 h1:P3g79IUS/93SYhtoeaHW+kRCIrYaxJ27MFPv+7kaTOw= modernc.org/ccgo/v3 v3.16.13 h1:Mkgdzl46i5F/CNR/Kj80Ri59hC8TKAhZrYSaqvkwzUw= diff --git a/handlers/api.go b/handlers/api.go index ea3945efb2..b89f241190 100644 --- a/handlers/api.go +++ b/handlers/api.go @@ -15,8 +15,7 @@ import ( "eth2-exporter/types" "eth2-exporter/utils" "fmt" - "io/ioutil" - "math" + "io" "math/big" "net/http" "net/url" @@ -289,6 +288,7 @@ func ApiEpoch(w http.ResponseWriter, r *http.Request) { epoch = int64(services.LatestEpoch()) } + latestFinalizedEpoch := services.LatestFinalizedEpoch() if vars["epoch"] == "finalized" { epoch = int64(services.LatestFinalizedEpoch()) } @@ -303,12 +303,12 @@ func ApiEpoch(w http.ResponseWriter, r *http.Request) { return } - rows, err := db.ReaderDb.Query(`SELECT attestationscount, attesterslashingscount, averagevalidatorbalance, blockscount, depositscount, eligibleether, epoch, finalized, globalparticipationrate, proposerslashingscount, rewards_exported, totalvalidatorbalance, validatorscount, voluntaryexitscount, votedether, withdrawalcount, + rows, err := db.ReaderDb.Query(`SELECT attestationscount, attesterslashingscount, averagevalidatorbalance, blockscount, depositscount, eligibleether, epoch, (epoch <= $2) AS finalized, globalparticipationrate, proposerslashingscount, rewards_exported, totalvalidatorbalance, validatorscount, voluntaryexitscount, votedether, withdrawalcount, (SELECT COUNT(*) FROM blocks WHERE epoch = $1 AND status = '0') as scheduledblocks, (SELECT COUNT(*) FROM blocks WHERE epoch = $1 AND status = '1') as proposedblocks, (SELECT COUNT(*) FROM blocks WHERE epoch = $1 AND status = '2') as missedblocks, (SELECT COUNT(*) FROM blocks WHERE epoch = $1 AND status = '3') as orphanedblocks - FROM epochs WHERE epoch = $1`, epoch) + FROM epochs WHERE epoch = $1`, epoch, latestFinalizedEpoch) if err != nil { logger.WithError(err).Error("error retrieving epoch data") sendServerErrorResponse(w, r.URL.String(), "could not retrieve db results") @@ -828,7 +828,7 @@ func ApiDashboard(w http.ResponseWriter, r *http.Request) { j := json.NewEncoder(w) - body, err := ioutil.ReadAll(r.Body) + body, err := io.ReadAll(r.Body) if err != nil { utils.LogError(err, "reading body", 0) sendErrorResponse(w, r.URL.String(), "could not read body") @@ -858,6 +858,7 @@ func ApiDashboard(w http.ResponseWriter, r *http.Request) { var currentSyncCommittee []interface{} var nextSyncCommittee []interface{} var syncCommitteeStats *SyncCommitteesInfo + var proposalLuckStats *types.ApiProposalLuckResponse if getValidators { queryIndices, err := parseApiValidatorParamToIndices(parsedBody.IndicesOrPubKey, maxValidators) @@ -869,20 +870,20 @@ func ApiDashboard(w http.ResponseWriter, r *http.Request) { if len(queryIndices) > 0 { g.Go(func() error { start := time.Now() - validatorsData, err = validators(queryIndices) + validatorsData, err = getGeneralValidatorInfoForAppDashboard(queryIndices) elapsed := time.Since(start) if elapsed > 10*time.Second { - logger.Warnf("validators(%v) took longer than 10 sec", queryIndices) + logger.Warnf("getGeneralValidatorInfoForAppDashboard(%v) took longer than 10 sec", queryIndices) } return err }) g.Go(func() error { start := time.Now() - validatorEffectivenessData, err = validatorEffectiveness(epoch-1, queryIndices) + validatorEffectivenessData, err = getValidatorEffectiveness(epoch-1, queryIndices) elapsed := time.Since(start) if elapsed > 10*time.Second { - logger.Warnf("validatorEffectiveness(%v, %v) took longer than 10 sec", epoch-1, queryIndices) + logger.Warnf("getValidatorEffectiveness(%v, %v) took longer than 10 sec", epoch-1, queryIndices) } return err }) @@ -940,6 +941,16 @@ func ApiDashboard(w http.ResponseWriter, r *http.Request) { } return err }) + + g.Go(func() error { + start := time.Now() + proposalLuckStats, err = getProposalLuckStats(queryIndices) + elapsed := time.Since(start) + if elapsed > 10*time.Second { + logger.Warnf("getProposalLuck(%v, %v) took longer than 10 sec", queryIndices, epoch) + } + return err + }) } } @@ -990,7 +1001,8 @@ func ApiDashboard(w http.ResponseWriter, r *http.Request) { ExecutionPerformance: executionPerformance, CurrentSyncCommittee: currentSyncCommittee, NextSyncCommittee: nextSyncCommittee, - SyncCommitteesStats: *syncCommitteeStats, + SyncCommitteesStats: syncCommitteeStats, + ProposalLuckStats: proposalLuckStats, } sendOKResponse(j, r.URL.String(), []interface{}{data}) @@ -1164,9 +1176,14 @@ func getSyncCommitteeSlotsStatistics(validators []uint64, epoch uint64) (types.S // validator_stats is updated only once a day, everything missing has to be collected from bigtable (which is slower than validator_stats) // check when the last update to validator_stats was - lastExportedDay := services.LatestExportedStatisticDay() epochsPerDay := utils.EpochsPerDay() - lastExportedEpoch := ((lastExportedDay + 1) * epochsPerDay) - 1 + lastExportedEpoch := uint64(0) + lastExportedDay, err := services.LatestExportedStatisticDay() + if err != nil && err != db.ErrNoStats { + return types.SyncCommitteesStats{}, fmt.Errorf("error retrieving latest exported statistics day: %v", err) + } else if err == nil { + lastExportedEpoch = ((lastExportedDay + 1) * epochsPerDay) - 1 + } // if epoch is not yet exported, we may need to collect the data from bigtable if lastExportedEpoch < epoch { @@ -1314,8 +1331,14 @@ func getRocketpoolValidators(queryIndices []uint64) ([]interface{}, error) { return utils.SqlRowsToJSON(rows) } -func validators(queryIndices []uint64) ([]interface{}, error) { +func getGeneralValidatorInfoForAppDashboard(queryIndices []uint64) ([]interface{}, error) { + // we use MAX(validatorindex)+1 instead of COUNT(*) for querying the rank_count for performance-reasons rows, err := db.ReaderDb.Query(` + WITH maxValidatorIndex AS ( + SELECT MAX(validatorindex)+1 as total_count + FROM validator_performance + WHERE validatorindex >= 0 AND validatorindex < 2147483647 + ) SELECT validators.validatorindex, pubkey, @@ -1332,7 +1355,8 @@ func validators(queryIndices []uint64) ([]interface{}, error) { COALESCE(validator_performance.cl_performance_31d, 0) AS performance31d, COALESCE(validator_performance.cl_performance_365d, 0) AS performance365d, COALESCE(validator_performance.cl_performance_total, 0) AS performanceTotal, - rank7d, + COALESCE(validator_performance.rank7d, 0) AS rank7d, + ((validator_performance.rank7d::float * 100) / COALESCE((SELECT total_count FROM maxValidatorIndex), 1)) as rankpercentage, w.total as total_withdrawals FROM validators LEFT JOIN validator_performance ON validators.validatorindex = validator_performance.validatorindex @@ -1356,20 +1380,40 @@ func validators(queryIndices []uint64) ([]interface{}, error) { return nil, fmt.Errorf("error converting validators to json: %w", err) } - balances, err := db.BigtableClient.GetValidatorBalanceHistory(queryIndices, services.LatestEpoch(), services.LatestEpoch()) - if err != nil { - return nil, fmt.Errorf("error getting validator balances from bigtable: %w", err) - } + g := new(errgroup.Group) - currentDayIncome, _, err := db.GetCurrentDayClIncome(queryIndices) - if err != nil { - return nil, err - } + var balances map[uint64][]*types.ValidatorBalance + g.Go(func() error { + balances, err = db.BigtableClient.GetValidatorBalanceHistory(queryIndices, services.LatestEpoch(), services.LatestEpoch()) + if err != nil { + return fmt.Errorf("error in GetValidatorBalanceHistory: %w", err) + } + return nil + }) - lastAttestationSlots, err := db.BigtableClient.GetLastAttestationSlots(queryIndices) + var currentDayIncome map[uint64]int64 + g.Go(func() error { + currentDayIncome, err = db.GetCurrentDayClIncome(queryIndices) + if err != nil { + return fmt.Errorf("error in GetCurrentDayClIncome: %w", err) + } + return nil + }) + + var lastAttestationSlots map[uint64]uint64 + g.Go(func() error { + lastAttestationSlots, err = db.BigtableClient.GetLastAttestationSlots(queryIndices) + if err != nil { + return fmt.Errorf("error in GetLastAttestationSlots: %w", err) + } + return nil + }) + + err = g.Wait() if err != nil { - return nil, fmt.Errorf("error getting validator last attestation slots from bigtable: %w", err) + return nil, fmt.Errorf("error in validator errgroup: %w", err) } + for _, entry := range data { eMap, ok := entry.(map[string]interface{}) if !ok { @@ -1405,7 +1449,7 @@ func validators(queryIndices []uint64) ([]interface{}, error) { return data, nil } -func validatorEffectiveness(epoch uint64, indices []uint64) ([]*types.ValidatorEffectiveness, error) { +func getValidatorEffectiveness(epoch uint64, indices []uint64) ([]*types.ValidatorEffectiveness, error) { data, err := db.BigtableClient.GetValidatorEffectiveness(indices, epoch) if err != nil { return nil, fmt.Errorf("error getting validator effectiveness from bigtable: %w", err) @@ -1432,14 +1476,16 @@ type DashboardResponse struct { ExecutionPerformance []types.ExecutionPerformanceResponse `json:"execution_performance"` CurrentSyncCommittee interface{} `json:"current_sync_committee"` NextSyncCommittee interface{} `json:"next_sync_committee"` - SyncCommitteesStats SyncCommitteesInfo `json:"sync_committees_stats"` + SyncCommitteesStats *SyncCommitteesInfo `json:"sync_committees_stats"` + ProposalLuckStats *types.ApiProposalLuckResponse `json:"proposal_luck_stats"` } func getEpoch(epoch int64) ([]interface{}, error) { + latestFinalizedEpoch := services.LatestFinalizedEpoch() rows, err := db.ReaderDb.Query(`SELECT attestationscount, attesterslashingscount, averagevalidatorbalance, - blockscount, depositscount, eligibleether, epoch, finalized, TRUNC(globalparticipationrate::decimal, 10)::float as globalparticipationrate, proposerslashingscount, + blockscount, depositscount, eligibleether, epoch, (epoch <= $2) AS finalized, TRUNC(globalparticipationrate::decimal, 10)::float as globalparticipationrate, proposerslashingscount, totalvalidatorbalance, validatorscount, voluntaryexitscount, votedether - FROM epochs WHERE epoch = $1`, epoch) + FROM epochs WHERE epoch = $1`, epoch, latestFinalizedEpoch) if err != nil { return nil, fmt.Errorf("error querying epoch: %w", err) } @@ -1450,7 +1496,7 @@ func getEpoch(epoch int64) ([]interface{}, error) { // ApiValidator godoc // @Summary Get up to 100 validators // @Tags Validator -// @Description Searching for too many validators based on their pubkeys will lead to an "URI too long" error +// @Description Searching for too many validators based on their pubkeys will lead to a "URI too long" error // @Produce json // @Param indexOrPubkey path string true "Up to 100 validator indicesOrPubkeys, comma separated" // @Success 200 {object} types.ApiResponse{data=[]types.APIValidatorResponse} @@ -1461,10 +1507,11 @@ func ApiValidatorGet(w http.ResponseWriter, r *http.Request) { } // ApiValidator godoc -// @Summary Get unlimited validators +// @Summary Get up to 100 validators // @Tags Validator +// @Description This POST endpoint exists because the GET endpoint can lead to a "URI too long" error when searching for too many validators based on their pubkeys. // @Produce json -// @Param indexOrPubkey body types.DashboardRequest true "Validator indicesOrPubkeys, comma separated" +// @Param indexOrPubkey body types.DashboardRequest true "Up to 100 validator indicesOrPubkeys, comma separated" // @Success 200 {object} types.ApiResponse{data=[]types.APIValidatorResponse} // @Failure 400 {object} types.ApiResponse // @Router /api/v1/validator [post] @@ -1478,16 +1525,13 @@ func apiValidator(w http.ResponseWriter, r *http.Request) { vars := mux.Vars(r) - var maxValidators int + maxValidators := getUserPremium(r).MaxValidators + var param string if r.Method == http.MethodGet { - maxValidators = getUserPremium(r).MaxValidators - // Get the validators from the URL param = vars["indexOrPubkey"] } else { - maxValidators = math.MaxInt - // Get the validators from the request body decoder := json.NewDecoder(r.Body) req := &types.DashboardRequest{} @@ -1578,6 +1622,7 @@ func apiValidator(w http.ResponseWriter, r *http.Request) { err = j.Encode(response) if err != nil { + sendServerErrorResponse(w, r.URL.String(), "could not serialize data results") logger.Errorf("error serializing json data for API %v route: %v", r.URL, err) } } @@ -1851,7 +1896,7 @@ func ApiValidatorIncomeDetailsHistory(w http.ResponseWriter, r *http.Request) { err = j.Encode(response) if err != nil { - sendErrorResponse(w, r.URL.String(), "could not serialize data results") + sendServerErrorResponse(w, r.URL.String(), "could not serialize data results") return } } @@ -1954,7 +1999,7 @@ func ApiValidatorWithdrawals(w http.ResponseWriter, r *http.Request) { err = json.NewEncoder(w).Encode(response) if err != nil { - sendErrorResponse(w, r.URL.String(), "could not serialize data results") + sendServerErrorResponse(w, r.URL.String(), "could not serialize data results") return } } @@ -2013,7 +2058,7 @@ func ApiValidatorBlsChange(w http.ResponseWriter, r *http.Request) { err = json.NewEncoder(w).Encode(response) if err != nil { - sendErrorResponse(w, r.URL.String(), "could not serialize data results") + sendServerErrorResponse(w, r.URL.String(), "could not serialize data results") return } } @@ -2092,7 +2137,7 @@ func ApiValidatorBalanceHistory(w http.ResponseWriter, r *http.Request) { err = j.Encode(response) if err != nil { - sendErrorResponse(w, r.URL.String(), "could not serialize data results") + sendServerErrorResponse(w, r.URL.String(), "could not serialize data results") return } } @@ -2177,7 +2222,7 @@ func ApiValidatorPerformance(w http.ResponseWriter, r *http.Request) { return } - currentDayIncome, _, err := db.GetCurrentDayClIncome(queryIndices) + currentDayIncome, err := db.GetCurrentDayClIncome(queryIndices) if err != nil { sendErrorResponse(w, r.URL.String(), "error retrieving current day income") return @@ -2296,7 +2341,7 @@ func ApiValidatorAttestationEffectiveness(w http.ResponseWriter, r *http.Request return } - data, err := validatorEffectiveness(services.LatestEpoch()-1, queryIndices) + data, err := getValidatorEffectiveness(services.LatestEpoch()-1, queryIndices) if err != nil { sendErrorResponse(w, r.URL.String(), "could not retrieve db results") return @@ -2310,7 +2355,7 @@ func ApiValidatorAttestationEffectiveness(w http.ResponseWriter, r *http.Request err = j.Encode(response) if err != nil { - sendErrorResponse(w, r.URL.String(), "could not serialize data results") + sendServerErrorResponse(w, r.URL.String(), "could not serialize data results") return } } @@ -2338,7 +2383,7 @@ func ApiValidatorAttestationEfficiency(w http.ResponseWriter, r *http.Request) { return } - data, err := validatorEffectiveness(services.LatestEpoch()-1, queryIndices) + data, err := getValidatorEffectiveness(services.LatestEpoch()-1, queryIndices) if err != nil { sendErrorResponse(w, r.URL.String(), "could not retrieve db results") return @@ -2352,7 +2397,7 @@ func ApiValidatorAttestationEfficiency(w http.ResponseWriter, r *http.Request) { err = j.Encode(response) if err != nil { - sendErrorResponse(w, r.URL.String(), "could not serialize data results") + sendServerErrorResponse(w, r.URL.String(), "could not serialize data results") return } } @@ -2504,7 +2549,7 @@ func ApiValidatorAttestations(w http.ResponseWriter, r *http.Request) { err = j.Encode(response) if err != nil { - sendErrorResponse(w, r.URL.String(), "could not serialize data results") + sendServerErrorResponse(w, r.URL.String(), "could not serialize data results") return } } @@ -3213,7 +3258,7 @@ func GetMobileWidgetStats(w http.ResponseWriter, r *http.Request, indexOrPubkey return } - currentDayIncome, _, err := db.GetCurrentDayClIncome(queryIndices) + currentDayIncome, err := db.GetCurrentDayClIncome(queryIndices) if err != nil { sendErrorResponse(w, r.URL.String(), "error retrieving current day income") return @@ -3248,7 +3293,7 @@ func GetMobileWidgetStats(w http.ResponseWriter, r *http.Request, indexOrPubkey } } - efficiencyData, err := validatorEffectiveness(services.LatestEpoch()-1, queryIndices) + efficiencyData, err := getValidatorEffectiveness(services.LatestEpoch()-1, queryIndices) if err != nil { sendErrorResponse(w, r.URL.String(), "could not parse db results") return @@ -3492,7 +3537,7 @@ func clientStatsPost(w http.ResponseWriter, r *http.Request, apiKey, machine str return } - body, err := ioutil.ReadAll(r.Body) + body, err := io.ReadAll(r.Body) if err != nil { logger.Warnf("error reading body | err: %v", err) sendErrorResponse(w, r.URL.String(), "could not read body") @@ -3714,6 +3759,115 @@ func ApiWithdrawalCredentialsValidators(w http.ResponseWriter, r *http.Request) sendOKResponse(json.NewEncoder(w), r.URL.String(), []interface{}{response}) } +// ApiProposalLuck godoc +// @Summary Get the proposal luck of a validator or a list of validators +// @Tags Validator +// @Description Returns the proposal luck of a validator or a list of validators +// @Produce json +// @Param validators query string true "Provide a comma separated list of validator indices or pubkeys" +// @Success 200 {object} types.ApiResponse{data=[]types.ApiProposalLuckResponse} +// @Failure 400 {object} types.ApiResponse +// @Failure 500 {object} types.ApiResponse +// @Router /api/v1/validators/proposalLuck [get] +func ApiProposalLuck(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + q := r.URL.Query() + response := &types.ApiResponse{} + response.Status = "OK" + + indices, pubkeys, err := parseValidatorsFromQueryString(q.Get("validators"), 100) + if err != nil { + sendErrorResponse(w, r.URL.String(), "could not parse validators") + return + } + if len(pubkeys) > 0 { + indicesFromPubkeys, err := resolveIndices(pubkeys) + if err != nil { + sendErrorResponse(w, r.URL.String(), "could not resolve pubkeys to indices") + return + } + indices = append(indices, indicesFromPubkeys...) + } + + if len(indices) == 0 { + sendErrorResponse(w, r.URL.String(), "no validators provided") + return + } + + // dedup indices + allKeys := make(map[uint64]bool) + list := []uint64{} + for _, item := range indices { + if _, ok := allKeys[item]; !ok { + allKeys[item] = true + list = append(list, item) + } + } + indices = list + data, err := getProposalLuckStats(indices) + if err != nil { + sendServerErrorResponse(w, r.URL.String(), "error processing request, please try again later") + utils.LogError(err, "error retrieving data from db for proposal luck", 0, map[string]interface{}{"request": r.Method + " " + r.URL.String()}) + } + + response.Data = data + err = json.NewEncoder(w).Encode(response) + if err != nil { + sendServerErrorResponse(w, r.URL.String(), "could not serialize data results") + utils.LogError(err, "error serializing json data for API", 0, map[string]interface{}{"request": r.Method + " " + r.URL.String()}) + } +} + +func getProposalLuckStats(indices []uint64) (*types.ApiProposalLuckResponse, error) { + data := types.ApiProposalLuckResponse{} + g := errgroup.Group{} + + var firstActivationEpoch uint64 + g.Go(func() error { + return db.GetFirstActivationEpoch(indices, &firstActivationEpoch) + }) + + var slots []uint64 + g.Go(func() error { + return db.ReaderDb.Select(&slots, ` + SELECT + slot + FROM blocks + WHERE proposer = ANY($1) + AND exec_block_number IS NOT NULL + ORDER BY slot ASC`, pq.Array(indices)) + }) + + err := g.Wait() + if err != nil { + return nil, err + } + + proposalLuck, proposalTimeFrame := getProposalLuck(slots, len(indices), firstActivationEpoch) + if proposalLuck > 0 { + data.ProposalLuck = &proposalLuck + timeframeName := getProposalTimeframeName(proposalTimeFrame) + data.TimeFrameName = &timeframeName + } + + avgProposalInterval := getAvgSlotInterval(len(indices)) + data.AverageProposalInterval = avgProposalInterval + + var estimateLowerBoundSlot *uint64 + if len(slots) > 0 { + estimateLowerBoundSlot = &slots[len(slots)-1] + } else if len(indices) == 1 { + activationSlot := firstActivationEpoch * utils.Config.Chain.ClConfig.SlotsPerEpoch + estimateLowerBoundSlot = &activationSlot + } + + if estimateLowerBoundSlot != nil { + nextProposalEstimate := utils.SlotToTime(*estimateLowerBoundSlot + uint64(avgProposalInterval)).Unix() + data.NextProposalEstimateTs = &nextProposalEstimate + } + return &data, nil +} + func DecodeMapStructure(input interface{}, output interface{}) error { config := &mapstructure.DecoderConfig{ Metadata: nil, @@ -3803,7 +3957,7 @@ func APIDashboardDataBalance(w http.ResponseWriter, r *http.Request) { err = json.NewEncoder(w).Encode(balanceHistoryChartData) if err != nil { logger.WithError(err).WithField("route", r.URL.String()).Error("error enconding json response") - http.Error(w, "Internal server error", http.StatusServiceUnavailable) + sendServerErrorResponse(w, r.URL.String(), "could not serialize data results") return } } @@ -3864,6 +4018,7 @@ func returnQueryResultsAsArray(rows *sql.Rows, w http.ResponseWriter, r *http.Re err = json.NewEncoder(w).Encode(response) if err != nil { + sendServerErrorResponse(w, r.URL.String(), "could not serialize data results") logger.Errorf("error serializing json data for API %v route: %v", r.URL.String(), err) } } diff --git a/handlers/api_eth1.go b/handlers/api_eth1.go index 32dff595b2..ea620d9bab 100644 --- a/handlers/api_eth1.go +++ b/handlers/api_eth1.go @@ -959,8 +959,13 @@ func getValidatorExecutionPerformance(queryIndices []uint64) ([]types.ExecutionP } } - lastStatsDay := services.LatestExportedStatisticDay() - firstEpochTime := utils.EpochToTime((lastStatsDay + 1) * utils.EpochsPerDay()) + firstEpochTime := utils.EpochToTime(0) + lastStatsDay, err := services.LatestExportedStatisticDay() + if err != nil && err != db.ErrNoStats { + return nil, fmt.Errorf("error retrieving latest exported statistics day: %v", err) + } else if err == nil { + firstEpochTime = utils.EpochToTime((lastStatsDay + 1) * utils.EpochsPerDay()) + } for _, block := range blocks { proposer := blockToProposerMap[block.Number].Proposer diff --git a/handlers/auth.go b/handlers/auth.go index 9002d1ebe1..4836ade5f1 100644 --- a/handlers/auth.go +++ b/handlers/auth.go @@ -156,12 +156,30 @@ func Login(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", "text/html") + q := r.URL.Query() + data := InitPageData(w, r, "login", "/login", "Login", templateFiles) - data.Data = types.AuthData{ + + authData := types.AuthData{ Flashes: utils.GetFlashes(w, r, authSessionName), CsrfField: csrf.TemplateField(r), RecaptchaKey: utils.Config.Frontend.RecaptchaSiteKey, } + + redirectData := struct { + Redirect_uri string + State string + }{ + Redirect_uri: q.Get("redirect_uri"), + State: q.Get("state"), + } + + data.Data = struct { + AuthData types.AuthData + RedirectData interface{} + }{ + AuthData: authData, + RedirectData: redirectData} data.Meta.NoTrack = true if handleTemplateError(w, r, "auth.go", "Login", "", loginTemplate.ExecuteTemplate(w, "layout", data)) != nil { @@ -171,7 +189,6 @@ func Login(w http.ResponseWriter, r *http.Request) { // LoginPost handles authenticating the user. func LoginPost(w http.ResponseWriter, r *http.Request) { - if err := utils.HandleRecaptcha(w, r, "/login"); err != nil { return } @@ -213,6 +230,17 @@ func LoginPost(w http.ResponseWriter, r *http.Request) { UserGroup string `db:"user_group"` }{} + redirectParam := "" + redirectURI := r.FormValue("oauth_redirect_uri") + if redirectURI != "" { + redirectParam = "?redirect_uri=" + redirectURI + + state := r.FormValue("state") + if state != "" { + redirectParam += "&state=" + state + } + } + err = db.FrontendWriterDB.Get(&user, "SELECT users.id, email, password, email_confirmed, COALESCE(product_id, '') as product_id, COALESCE(active, false) as active, COALESCE(user_group, '') AS user_group FROM users left join users_app_subscriptions on users_app_subscriptions.user_id = users.id WHERE email = $1", email) if err != nil { if err != sql.ErrNoRows { @@ -220,14 +248,14 @@ func LoginPost(w http.ResponseWriter, r *http.Request) { } session.AddFlash("Error: Invalid email or password!") session.Save(r, w) - http.Redirect(w, r, "/login", http.StatusSeeOther) + http.Redirect(w, r, "/login"+redirectParam, http.StatusSeeOther) return } if !user.Confirmed { session.AddFlash("Error: Email has not been confirmed, please click the link in the email we sent you or resend link!") session.Save(r, w) - http.Redirect(w, r, "/login", http.StatusSeeOther) + http.Redirect(w, r, "/login"+redirectParam, http.StatusSeeOther) return } @@ -235,7 +263,7 @@ func LoginPost(w http.ResponseWriter, r *http.Request) { if err != nil { session.AddFlash("Error: Invalid email or password!") session.Save(r, w) - http.Redirect(w, r, "/login", http.StatusSeeOther) + http.Redirect(w, r, "/login"+redirectParam, http.StatusSeeOther) return } @@ -282,20 +310,8 @@ func LoginPost(w http.ResponseWriter, r *http.Request) { }, ).Info("login succeeded") - redirectURI := session.GetValue("oauth_redirect_uri") - - if redirectURI != nil { - state := session.GetValue("state") - var stateParam = "" - - if state != nil { - stateParam = "&state=" + state.(string) - } - - session.DeleteValue("oauth_redirect_uri") - session.DeleteValue("state") - - http.Redirect(w, r, "/user/authorize?redirect_uri="+redirectURI.(string)+stateParam, http.StatusSeeOther) + if redirectParam != "" { + http.Redirect(w, r, "/user/authorize"+redirectParam, http.StatusSeeOther) return } @@ -316,7 +332,6 @@ func Logout(w http.ResponseWriter, r *http.Request) { session.SetValue("subscription", "") session.SetValue("authenticated", false) session.DeleteValue("user_id") - session.DeleteValue("oauth_redirect_uri") err = session.SCS.Destroy(r.Context()) if err != nil { diff --git a/handlers/charts.go b/handlers/charts.go index 3685eab43b..8db3c05a16 100644 --- a/handlers/charts.go +++ b/handlers/charts.go @@ -100,7 +100,7 @@ func GenericChart(w http.ResponseWriter, r *http.Request) { } if chartData == nil { - http.Error(w, "Not found", http.StatusNotFound) + NotFound(w, r) return } diff --git a/handlers/common.go b/handlers/common.go index 451defd6ce..c86da98a6d 100644 --- a/handlers/common.go +++ b/handlers/common.go @@ -12,6 +12,7 @@ import ( "eth2-exporter/utils" "fmt" "html/template" + "math" "math/big" "net/http" "strconv" @@ -47,8 +48,13 @@ func GetValidatorEarnings(validators []uint64, currency string) (*types.Validato return nil, nil, errors.New("no validators provided") } latestFinalizedEpoch := services.LatestFinalizedEpoch() - lastStatsDay := services.LatestExportedStatisticDay() - firstSlot := utils.GetLastBalanceInfoSlotForDay(lastStatsDay) + 1 + + firstSlot := uint64(0) + lastStatsDay, lastExportedStatsErr := services.LatestExportedStatisticDay() + if lastExportedStatsErr == nil { + firstSlot = utils.GetLastBalanceInfoSlotForDay(lastStatsDay) + 1 + } + lastSlot := latestFinalizedEpoch * utils.Config.Chain.ClConfig.SlotsPerEpoch balancesMap := make(map[uint64]*types.Validator, 0) @@ -80,7 +86,7 @@ func GetValidatorEarnings(validators []uint64, currency string) (*types.Validato income := types.ValidatorIncomePerformance{} g.Go(func() error { - return db.GetValidatorIncomePerforamance(validators, &income) + return db.GetValidatorIncomePerformance(validators, &income) }) var totalDeposits uint64 @@ -99,18 +105,25 @@ func GetValidatorEarnings(validators []uint64, currency string) (*types.Validato }) var lastDeposits uint64 - g.Go(func() error { - return db.GetValidatorDepositsForSlots(validators, firstSlot, lastSlot, &lastDeposits) - }) - var lastWithdrawals uint64 - g.Go(func() error { - return db.GetValidatorWithdrawalsForSlots(validators, firstSlot, lastSlot, &lastWithdrawals) - }) - var lastBalance uint64 g.Go(func() error { - return db.GetValidatorBalanceForDay(validators, lastStatsDay, &lastBalance) + if lastExportedStatsErr == db.ErrNoStats { + err := db.GetValidatorActivationBalance(validators, &lastBalance) + if err != nil { + return err + } + } else { + err := db.GetValidatorBalanceForDay(validators, lastStatsDay, &lastBalance) + if err != nil { + return err + } + } + err := db.GetValidatorDepositsForSlots(validators, firstSlot, lastSlot, &lastDeposits) + if err != nil { + return err + } + return db.GetValidatorWithdrawalsForSlots(validators, firstSlot, lastSlot, &lastWithdrawals) }) proposals := []types.ValidatorProposalInfo{} @@ -137,35 +150,48 @@ func GetValidatorEarnings(validators []uint64, currency string) (*types.Validato if clApr7d < float64(-1) { clApr7d = float64(-1) } + if math.IsNaN(clApr7d) { + clApr7d = float64(0) + } elApr7d := ((float64(income.ElIncome7d) / float64(totalDeposits)) * 365) / 7 if elApr7d < float64(-1) { elApr7d = float64(-1) } + if math.IsNaN(elApr7d) { + elApr7d = float64(0) + } clApr31d := ((float64(income.ClIncome31d) / float64(totalDeposits)) * 365) / 31 if clApr31d < float64(-1) { clApr31d = float64(-1) } + if math.IsNaN(clApr31d) { + clApr31d = float64(0) + } elApr31d := ((float64(income.ElIncome31d) / float64(totalDeposits)) * 365) / 31 if elApr31d < float64(-1) { elApr31d = float64(-1) } + if math.IsNaN(elApr31d) { + elApr31d = float64(0) + } + clApr365d := (float64(income.ClIncome365d) / float64(totalDeposits)) if clApr365d < float64(-1) { clApr365d = float64(-1) } + if math.IsNaN(clApr365d) { + clApr365d = float64(0) + } elApr365d := (float64(income.ElIncome365d) / float64(totalDeposits)) if elApr365d < float64(-1) { elApr365d = float64(-1) } - - // retrieve cl income not yet in stats - currentDayProposerIncome, err := db.GetCurrentDayProposerIncomeTotal(validators) - if err != nil { - return nil, nil, err + if math.IsNaN(elApr365d) { + elApr365d = float64(0) } incomeToday := types.ClElInt64{ @@ -175,7 +201,10 @@ func GetValidatorEarnings(validators []uint64, currency string) (*types.Validato } proposedToday := []uint64{} - todayStartEpoch := uint64(lastStatsDay+1) * utils.EpochsPerDay() + todayStartEpoch := uint64(0) + if lastExportedStatsErr == nil { + todayStartEpoch = uint64(lastStatsDay+1) * utils.EpochsPerDay() + } validatorProposalData := types.ValidatorProposalData{} validatorProposalData.Proposals = make([][]uint64, len(proposals)) for i, b := range proposals { @@ -213,8 +242,8 @@ func GetValidatorEarnings(validators []uint64, currency string) (*types.Validato } } - validatorProposalData.ProposalLuck = getProposalLuck(slots, len(validators), firstActivationEpoch) - avgSlotInterval := uint64(getAvgSlotInterval(1)) + validatorProposalData.ProposalLuck, _ = getProposalLuck(slots, len(validators), firstActivationEpoch) + avgSlotInterval := uint64(getAvgSlotInterval(len(validators))) avgSlotIntervalAsDuration := time.Duration(utils.Config.Chain.ClConfig.SecondsPerSlot*avgSlotInterval) * time.Second validatorProposalData.AvgSlotInterval = &avgSlotIntervalAsDuration if len(slots) > 0 { @@ -259,12 +288,6 @@ func GetValidatorEarnings(validators []uint64, currency string) (*types.Validato Total: income.ClIncomeTotal + income.ElIncomeTotal + incomeToday.Total, } - incomeTotalProposer := types.ClElInt64{ - El: income.ElIncomeTotal, - Cl: income.ClProposerIncomeTotal + currentDayProposerIncome, - Total: income.ClProposerIncomeTotal + income.ElIncomeTotal + currentDayProposerIncome, - } - return &types.ValidatorEarnings{ Income1d: types.ClElInt64{ El: income.ElIncome1d, @@ -298,45 +321,45 @@ func GetValidatorEarnings(validators []uint64, currency string) (*types.Validato Cl: clApr365d, Total: clApr365d + elApr365d, }, - TotalDeposits: int64(totalDeposits), - LastDayFormatted: utils.FormatIncome(earnings1d, currency), - LastWeekFormatted: utils.FormatIncome(earnings7d, currency), - LastMonthFormatted: utils.FormatIncome(earnings31d, currency), - TotalFormatted: utils.FormatIncomeClElInt64(incomeTotal, currency), - ProposerTotalFormatted: utils.FormatIncomeClElInt64(incomeTotalProposer, currency), - TotalChangeFormatted: utils.FormatIncome(income.ClIncomeTotal+currentDayClIncome+int64(totalDeposits), currency), - TotalBalance: utils.FormatIncome(int64(totalBalance), currency), - ProposalData: validatorProposalData, + TotalDeposits: int64(totalDeposits), + LastDayFormatted: utils.FormatIncome(earnings1d, currency), + LastWeekFormatted: utils.FormatIncome(earnings7d, currency), + LastMonthFormatted: utils.FormatIncome(earnings31d, currency), + TotalFormatted: utils.FormatIncomeClElInt64(incomeTotal, currency), + TotalChangeFormatted: utils.FormatIncome(income.ClIncomeTotal+currentDayClIncome+int64(totalDeposits), currency), + TotalBalance: utils.FormatIncome(int64(totalBalance), currency), + ProposalData: validatorProposalData, }, balancesMap, nil } +// Timeframe constants +const fiveDays = utils.Day * 5 +const oneWeek = utils.Week +const oneMonth = utils.Month +const sixWeeks = utils.Day * 45 +const twoMonths = utils.Month * 2 +const threeMonths = utils.Month * 3 +const fourMonths = utils.Month * 4 +const fiveMonths = utils.Month * 5 +const sixMonths = utils.Month * 6 +const year = utils.Year + // getProposalLuck calculates the luck of a given set of proposed blocks for a certain number of validators // given the blocks proposed by the validators and the number of validators // // precondition: slots is sorted by ascending block number -func getProposalLuck(slots []uint64, validatorsCount int, fromEpoch uint64) float64 { +func getProposalLuck(slots []uint64, validatorsCount int, fromEpoch uint64) (float64, time.Duration) { // Return 0 if there are no proposed blocks or no validators if len(slots) == 0 || validatorsCount == 0 { - return 0 + return 0, 0 } - // Timeframe constants - fiveDays := utils.Day * 5 - oneWeek := utils.Week - oneMonth := utils.Month - sixWeeks := utils.Day * 45 - twoMonths := utils.Month * 2 - threeMonths := utils.Month * 3 - fourMonths := utils.Month * 4 - fiveMonths := utils.Month * 5 - sixMonths := utils.Month * 6 - year := utils.Year activeValidatorsCount := *services.GetLatestStats().ActiveValidatorCount // Calculate the expected number of slot proposals for 30 days expectedSlotProposals := calcExpectedSlotProposals(oneMonth, validatorsCount, activeValidatorsCount) // Get the timeframe for which we should consider qualified proposals - var proposalTimeframe time.Duration + var proposalTimeFrame time.Duration // Time since the first epoch of the related validators timeSinceFirstEpoch := time.Since(utils.EpochToTime(fromEpoch)) @@ -345,36 +368,36 @@ func getProposalLuck(slots []uint64, validatorsCount int, fromEpoch uint64) floa // Determine the appropriate timeframe based on the time since the first block and the expected slot proposals switch { case timeSinceFirstEpoch < fiveDays: - proposalTimeframe = fiveDays + proposalTimeFrame = fiveDays case timeSinceFirstEpoch < oneWeek: - proposalTimeframe = oneWeek + proposalTimeFrame = oneWeek case timeSinceFirstEpoch < oneMonth: - proposalTimeframe = oneMonth + proposalTimeFrame = oneMonth case timeSinceFirstEpoch > year && expectedSlotProposals <= targetBlocks/12: - proposalTimeframe = year + proposalTimeFrame = year case timeSinceFirstEpoch > sixMonths && expectedSlotProposals <= targetBlocks/6: - proposalTimeframe = sixMonths + proposalTimeFrame = sixMonths case timeSinceFirstEpoch > fiveMonths && expectedSlotProposals <= targetBlocks/5: - proposalTimeframe = fiveMonths + proposalTimeFrame = fiveMonths case timeSinceFirstEpoch > fourMonths && expectedSlotProposals <= targetBlocks/4: - proposalTimeframe = fourMonths + proposalTimeFrame = fourMonths case timeSinceFirstEpoch > threeMonths && expectedSlotProposals <= targetBlocks/3: - proposalTimeframe = threeMonths + proposalTimeFrame = threeMonths case timeSinceFirstEpoch > twoMonths && expectedSlotProposals <= targetBlocks/2: - proposalTimeframe = twoMonths + proposalTimeFrame = twoMonths case timeSinceFirstEpoch > sixWeeks && expectedSlotProposals <= targetBlocks/1.5: - proposalTimeframe = sixWeeks + proposalTimeFrame = sixWeeks default: - proposalTimeframe = oneMonth + proposalTimeFrame = oneMonth } // Recalculate expected slot proposals for the new timeframe - expectedSlotProposals = calcExpectedSlotProposals(proposalTimeframe, validatorsCount, activeValidatorsCount) + expectedSlotProposals = calcExpectedSlotProposals(proposalTimeFrame, validatorsCount, activeValidatorsCount) if expectedSlotProposals == 0 { - return 0 + return 0, 0 } // Cutoff time for proposals to be considered qualified - blockProposalCutoffTime := time.Now().Add(-proposalTimeframe) + blockProposalCutoffTime := time.Now().Add(-proposalTimeFrame) // Count the number of qualified proposals qualifiedProposalCount := 0 @@ -384,7 +407,34 @@ func getProposalLuck(slots []uint64, validatorsCount int, fromEpoch uint64) floa } } // Return the luck as the ratio of qualified proposals to expected slot proposals - return float64(qualifiedProposalCount) / expectedSlotProposals + return float64(qualifiedProposalCount) / expectedSlotProposals, proposalTimeFrame +} + +func getProposalTimeframeName(proposalTimeframe time.Duration) string { + switch { + case proposalTimeframe == fiveDays: + return "5 days" + case proposalTimeframe == oneWeek: + return "week" + case proposalTimeframe == oneMonth: + return "month" + case proposalTimeframe == sixWeeks: + return "6 weeks" + case proposalTimeframe == twoMonths: + return "2 months" + case proposalTimeframe == threeMonths: + return "3 months" + case proposalTimeframe == fourMonths: + return "4 months" + case proposalTimeframe == fiveMonths: + return "5 months" + case proposalTimeframe == sixMonths: + return "6 months" + case proposalTimeframe == year: + return "year" + default: + return "month" + } } // calcExpectedSlotProposals calculates the expected number of slot proposals for a certain time frame and validator count @@ -606,8 +656,8 @@ func SetDataTableStateChanges(w http.ResponseWriter, r *http.Request) { settings := types.DataTableSaveState{} err = json.NewDecoder(r.Body).Decode(&settings) if err != nil { - utils.LogError(err, errMsgPrefix+", could not parse body", 0, errFields) - w.WriteHeader(http.StatusInternalServerError) + logger.Warnf(errMsgPrefix+", could not parse body for tableKey %v: %v", tableKey, err) + w.WriteHeader(http.StatusBadRequest) return } diff --git a/handlers/correlations.go b/handlers/correlations.go index 70d5c388e9..fcdad98df8 100644 --- a/handlers/correlations.go +++ b/handlers/correlations.go @@ -29,20 +29,6 @@ func Correlations(w http.ResponseWriter, r *http.Request) { var correlationsTemplate = templates.GetTemplate(templateFiles...) - // data := &types.PageData{ - // Meta: &types.Meta{ - // Image: "https://etherchain.org/img/ballon-512x512.png", - // Title: fmt.Sprintf("%v - Correlations - etherchain.org - %v", utils.Config.Frontend.SiteName, time.Now().Year()), - // Description: "etherchain.org makes the Ethereum block chain accessible to non-technical end users", - // Path: "/correlations", - // GATag: utils.Config.Frontend.GATag, - // }, - // Active: "stats", - // Data: indicators, - // CurrentBlock: services.LatestBlock(), - // GPO: services.LatestGasNowData(), - // } - if handleTemplateError(w, r, "correlations.go", "Correlations", "", correlationsTemplate.ExecuteTemplate(w, "layout", data)) != nil { return // an error has occurred and was processed } diff --git a/handlers/dashboard.go b/handlers/dashboard.go index 21ebca0e70..196bc803ca 100644 --- a/handlers/dashboard.go +++ b/handlers/dashboard.go @@ -450,12 +450,12 @@ func DashboardDataBalanceCombined(w http.ResponseWriter, r *http.Request) { if len(param) != 0 { days, err := strconv.ParseUint(param, 10, 32) if err != nil { - logger.Error(err) - http.Error(w, "Error: invalid days parameter", http.StatusBadRequest) + logger.Warnf("error parsing days: %v", err) + http.Error(w, "Error: invalid parameter days", http.StatusBadRequest) return } - lastStatsDay := services.LatestExportedStatisticDay() - if days < lastStatsDay { + lastStatsDay, err := services.LatestExportedStatisticDay() + if days < lastStatsDay && err == nil { lowerBoundDay = lastStatsDay - days + 1 } } @@ -648,14 +648,14 @@ func DashboardDataWithdrawals(w http.ResponseWriter, r *http.Request) { draw, err := strconv.ParseUint(q.Get("draw"), 10, 64) if err != nil { - utils.LogError(err, fmt.Errorf("error converting datatables data parameter from string to int: %v", err), 0) - http.Error(w, "Internal server error", http.StatusInternalServerError) + logger.Warnf("error converting datatables draw parameter from string to int: %v", err) + http.Error(w, "Error: Missing or invalid parameter draw", http.StatusBadRequest) return } start, err := strconv.ParseUint(q.Get("start"), 10, 64) if err != nil { - utils.LogError(err, fmt.Errorf("error converting datatables data parameter from string to int: %v", err), 0) - http.Error(w, "Internal server error", http.StatusInternalServerError) + logger.Warnf("error converting datatables start parameter from string to int: %v", err) + http.Error(w, "Error: Missing or invalid parameter start", http.StatusBadRequest) return } @@ -1023,7 +1023,12 @@ func DashboardDataEffectiveness(w http.ResponseWriter, r *http.Request) { var avgIncDistance []float64 - effectiveness, err := db.BigtableClient.GetValidatorEffectiveness(activeValidators, services.LatestEpoch()-1) + epoch := services.LatestEpoch() + if epoch > 0 { + epoch = epoch - 1 + } + + effectiveness, err := db.BigtableClient.GetValidatorEffectiveness(activeValidators, epoch) for _, e := range effectiveness { avgIncDistance = append(avgIncDistance, e.AttestationEfficiency) } @@ -1072,7 +1077,7 @@ func DashboardDataProposalsHistory(w http.ResponseWriter, r *http.Request) { } lastDay, err := db.GetLastExportedStatisticDay() - if err != nil { + if err != nil && err != db.ErrNoStats { logger.WithError(err).WithField("route", r.URL.String()).Error("error retrieving last exported statistic day") http.Error(w, "Internal server error", http.StatusServiceUnavailable) return diff --git a/handlers/education.go b/handlers/education.go deleted file mode 100644 index 5eb28d5147..0000000000 --- a/handlers/education.go +++ /dev/null @@ -1,19 +0,0 @@ -package handlers - -import ( - "eth2-exporter/templates" - "net/http" -) - -func EducationServices(w http.ResponseWriter, r *http.Request) { - templateFiles := append(layoutTemplateFiles, "educationServices.html") - var educationServicesTemplate = templates.GetTemplate(templateFiles...) - - w.Header().Set("Content-Type", "text/html") - - data := InitPageData(w, r, "services", "/educationServices", "Ethereum Education Services Overview", templateFiles) - - if handleTemplateError(w, r, "education.go", "EducationServices", "", educationServicesTemplate.ExecuteTemplate(w, "layout", data)) != nil { - return // an error has occurred and was processed - } -} diff --git a/handlers/epoch.go b/handlers/epoch.go index 603f65b686..6ae5e6fc12 100644 --- a/handlers/epoch.go +++ b/handlers/epoch.go @@ -4,10 +4,12 @@ import ( "database/sql" "encoding/json" "eth2-exporter/db" + "eth2-exporter/services" "eth2-exporter/templates" "eth2-exporter/types" "eth2-exporter/utils" "fmt" + "math" "net/http" "strconv" "strings" @@ -28,7 +30,7 @@ func Epoch(w http.ResponseWriter, r *http.Request) { var epochFutureTemplate = templates.GetTemplate(epochFutureTemplateFiles...) var epochNotFoundTemplate = templates.GetTemplate(epochNotFoundTemplateFiles...) - const MaxEpochValue = 4294967296 // we only render a page for epochs up to this value + const MaxEpochValue = math.MaxUint32 + 1 // we only render a page for epochs up to this value w.Header().Set("Content-Type", "text/html") vars := mux.Vars(r) @@ -48,6 +50,7 @@ func Epoch(w http.ResponseWriter, r *http.Request) { } epochPageData := types.EpochPageData{} + latestFinalizedEpoch := services.LatestFinalizedEpoch() err = db.ReaderDb.Get(&epochPageData, ` SELECT @@ -60,12 +63,12 @@ func Epoch(w http.ResponseWriter, r *http.Request) { voluntaryexitscount, validatorscount, averagevalidatorbalance, - finalized, + (epoch <= $2) AS finalized, eligibleether, globalparticipationrate, votedether FROM epochs - WHERE epoch = $1`, epoch) + WHERE epoch = $1`, epoch, latestFinalizedEpoch) if err != nil { //Epoch not in database -> Show future epoch if epoch > MaxEpochValue { diff --git a/handlers/epochs.go b/handlers/epochs.go index e70b949bdd..4b1d4e4e0c 100644 --- a/handlers/epochs.go +++ b/handlers/epochs.go @@ -42,20 +42,20 @@ func EpochsData(w http.ResponseWriter, r *http.Request) { draw, err := strconv.ParseUint(q.Get("draw"), 10, 64) if err != nil { - logger.Errorf("error converting datatables data parameter from string to int: %v", err) - http.Error(w, "Internal server error", http.StatusServiceUnavailable) + logger.Warnf("error converting datatables draw parameter from string to int: %v", err) + http.Error(w, "Error: Missing or invalid parameter draw", http.StatusBadRequest) return } start, err := strconv.ParseUint(q.Get("start"), 10, 64) if err != nil { - logger.Errorf("error converting datatables start parameter from string to int: %v", err) - http.Error(w, "Internal server error", http.StatusServiceUnavailable) + logger.Warnf("error converting datatables start parameter from string to int: %v", err) + http.Error(w, "Error: Missing or invalid parameter start", http.StatusBadRequest) return } length, err := strconv.ParseUint(q.Get("length"), 10, 64) if err != nil { - logger.Errorf("error converting datatables length parameter from string to int: %v", err) - http.Error(w, "Internal server error", http.StatusServiceUnavailable) + logger.Warnf("error converting datatables length parameter from string to int: %v", err) + http.Error(w, "Error: Missing or invalid parameter length", http.StatusBadRequest) return } if length > 100 { @@ -76,6 +76,7 @@ func EpochsData(w http.ResponseWriter, r *http.Request) { var epochs []*types.EpochsPageData + latestFinalizedEpoch := services.LatestFinalizedEpoch() if search == -1 { err = db.ReaderDb.Select(&epochs, ` SELECT epoch, @@ -88,13 +89,13 @@ func EpochsData(w http.ResponseWriter, r *http.Request) { voluntaryexitscount, validatorscount, averagevalidatorbalance, - finalized, + (epoch <= $3) AS finalized, eligibleether, globalparticipationrate, votedether FROM epochs WHERE epoch >= $1 AND epoch <= $2 - ORDER BY epoch DESC`, endEpoch, startEpoch) + ORDER BY epoch DESC`, endEpoch, startEpoch, latestFinalizedEpoch) } else { err = db.ReaderDb.Select(&epochs, ` SELECT epoch, @@ -106,13 +107,13 @@ func EpochsData(w http.ResponseWriter, r *http.Request) { voluntaryexitscount, validatorscount, averagevalidatorbalance, - finalized, + (epoch <= $2) AS finalized, eligibleether, globalparticipationrate, votedether FROM epochs WHERE epoch = $1 - ORDER BY epoch DESC`, search) + ORDER BY epoch DESC`, search, latestFinalizedEpoch) } if err != nil { logger.Errorf("error retrieving epoch data: %v", err) diff --git a/handlers/eth1Block.go b/handlers/eth1Block.go index 59dff5d8e0..2170012982 100644 --- a/handlers/eth1Block.go +++ b/handlers/eth1Block.go @@ -121,7 +121,7 @@ func Eth1Block(w http.ResponseWriter, r *http.Request) { func GetExecutionBlockPageData(number uint64, limit int) (*types.Eth1BlockPageData, error) { block, err := db.BigtableClient.GetBlockFromBlocksTable(number) if diffToHead := int64(services.LatestEth1BlockNumber()) - int64(number); err != nil && diffToHead < 0 && diffToHead >= -5 { - block, _, err = rpc.CurrentErigonClient.GetBlock(int64(number)) + block, _, err = rpc.CurrentErigonClient.GetBlock(int64(number), "parity/geth") } if err != nil { return nil, err diff --git a/handlers/eth1Blocks.go b/handlers/eth1Blocks.go index 5b3d81107e..5e24838fb6 100644 --- a/handlers/eth1Blocks.go +++ b/handlers/eth1Blocks.go @@ -42,20 +42,20 @@ func Eth1BlocksData(w http.ResponseWriter, r *http.Request) { } draw, err := strconv.ParseUint(q.Get("draw"), 10, 64) if err != nil { - logger.Errorf("error converting datatables data parameter from string to int for route %v: %v", r.URL.String(), err) - http.Error(w, "Internal server error", http.StatusServiceUnavailable) + logger.Warnf("error converting datatables draw parameter from string to int for route %v: %v", r.URL.String(), err) + http.Error(w, "Error: Missing or invalid parameter draw", http.StatusBadRequest) return } start, err := strconv.ParseUint(q.Get("start"), 10, 64) if err != nil { - logger.Errorf("error converting datatables start parameter from string to int for route %v: %v", r.URL.String(), err) - http.Error(w, "Internal server error", http.StatusServiceUnavailable) + logger.Warnf("error converting datatables start parameter from string to int for route %v: %v", r.URL.String(), err) + http.Error(w, "Error: Missing or invalid parameter start", http.StatusBadRequest) return } length, err := strconv.ParseUint(q.Get("length"), 10, 64) if err != nil { - logger.Errorf("error converting datatables length parameter from string to int for route %v: %v", r.URL.String(), err) - http.Error(w, "Internal server error", http.StatusServiceUnavailable) + logger.Warnf("error converting datatables length parameter from string to int for route %v: %v", r.URL.String(), err) + http.Error(w, "Error: Missing or invalid parameter length", http.StatusBadRequest) return } if length > 100 { diff --git a/handlers/eth1Deposits.go b/handlers/eth1Deposits.go index 847e185619..d5fe80c2f1 100644 --- a/handlers/eth1Deposits.go +++ b/handlers/eth1Deposits.go @@ -59,20 +59,20 @@ func Eth1DepositsData(w http.ResponseWriter, r *http.Request) { draw, err := strconv.ParseUint(q.Get("draw"), 10, 64) if err != nil { - logger.Errorf("error converting datatables data parameter from string to int: %v", err) - http.Error(w, "Internal server error", http.StatusServiceUnavailable) + logger.Warnf("error converting datatables draw parameter from string to int: %v", err) + http.Error(w, "Error: Missing or invalid parameter draw", http.StatusBadRequest) return } start, err := strconv.ParseUint(q.Get("start"), 10, 64) if err != nil { - logger.Errorf("error converting datatables start parameter from string to int: %v", err) - http.Error(w, "Internal server error", http.StatusServiceUnavailable) + logger.Warnf("error converting datatables start parameter from string to int: %v", err) + http.Error(w, "Error: Missing or invalid parameter start", http.StatusBadRequest) return } length, err := strconv.ParseUint(q.Get("length"), 10, 64) if err != nil { - logger.Errorf("error converting datatables length parameter from string to int: %v", err) - http.Error(w, "Internal server error", http.StatusServiceUnavailable) + logger.Warnf("error converting datatables length parameter from string to int: %v", err) + http.Error(w, "Error: Missing or invalid parameter length", http.StatusBadRequest) return } if length > 100 { @@ -171,20 +171,20 @@ func Eth1DepositsLeaderboardData(w http.ResponseWriter, r *http.Request) { draw, err := strconv.ParseUint(q.Get("draw"), 10, 64) if err != nil { - logger.Errorf("error converting datatables data parameter from string to int: %v", err) - http.Error(w, "Internal server error", http.StatusServiceUnavailable) + logger.Warnf("error converting datatables draw parameter from string to int: %v", err) + http.Error(w, "Error: Missing or invalid parameter draw", http.StatusBadRequest) return } start, err := strconv.ParseUint(q.Get("start"), 10, 64) if err != nil { - logger.Errorf("error converting datatables start parameter from string to int: %v", err) - http.Error(w, "Internal server error", http.StatusServiceUnavailable) + logger.Warnf("error converting datatables start parameter from string to int: %v", err) + http.Error(w, "Error: Missing or invalid parameter start", http.StatusBadRequest) return } length, err := strconv.ParseUint(q.Get("length"), 10, 64) if err != nil { - logger.Errorf("error converting datatables length parameter from string to int: %v", err) - http.Error(w, "Internal server error", http.StatusServiceUnavailable) + logger.Warnf("error converting datatables length parameter from string to int: %v", err) + http.Error(w, "Error: Missing or invalid parameter length", http.StatusBadRequest) return } if length > 100 { diff --git a/handlers/eth1tx.go b/handlers/eth1tx.go index 04eef72043..add941ddbf 100644 --- a/handlers/eth1tx.go +++ b/handlers/eth1tx.go @@ -42,7 +42,7 @@ func Eth1TransactionTx(w http.ResponseWriter, r *http.Request) { txHash, err := hex.DecodeString(strings.ReplaceAll(txHashString, "0x", "")) if err != nil { - logger.Errorf("error parsing tx hash %v: %v", txHashString, err) + logger.Warnf("error parsing tx hash %v: %v", txHashString, err) data = InitPageData(w, r, "blockchain", path, title, txNotFoundTemplateFiles) txTemplate = txNotFoundTemplate } else { diff --git a/handlers/eth2Deposits.go b/handlers/eth2Deposits.go index ae3883da90..032b1deb14 100644 --- a/handlers/eth2Deposits.go +++ b/handlers/eth2Deposits.go @@ -27,20 +27,20 @@ func Eth2DepositsData(w http.ResponseWriter, r *http.Request) { draw, err := strconv.ParseUint(q.Get("draw"), 10, 64) if err != nil { - logger.Errorf("error converting datatables data parameter from string to int: %v", err) - http.Error(w, "Internal server error", http.StatusServiceUnavailable) + logger.Warnf("error converting datatables draw parameter from string to int: %v", err) + http.Error(w, "Error: Missing or invalid parameter draw", http.StatusBadRequest) return } start, err := strconv.ParseUint(q.Get("start"), 10, 64) if err != nil { - logger.Errorf("error converting datatables start parameter from string to int: %v", err) - http.Error(w, "Internal server error", http.StatusServiceUnavailable) + logger.Warnf("error converting datatables start parameter from string to int: %v", err) + http.Error(w, "Error: Missing or invalid parameter start", http.StatusBadRequest) return } length, err := strconv.ParseUint(q.Get("length"), 10, 64) if err != nil { - logger.Errorf("error converting datatables length parameter from string to int: %v", err) - http.Error(w, "Internal server error", http.StatusServiceUnavailable) + logger.Warnf("error converting datatables length parameter from string to int: %v", err) + http.Error(w, "Error: Missing or invalid parameter length", http.StatusBadRequest) return } if length > 100 { diff --git a/handlers/faq.go b/handlers/faq.go deleted file mode 100644 index 7a92b7d5c3..0000000000 --- a/handlers/faq.go +++ /dev/null @@ -1,20 +0,0 @@ -package handlers - -import ( - "eth2-exporter/templates" - "net/http" -) - -// Faq will return the data from the frequently asked questions (FAQ) using a go template -func Faq(w http.ResponseWriter, r *http.Request) { - templateFiles := append(layoutTemplateFiles, "faq.html") - var faqTemplate = templates.GetTemplate(templateFiles...) - - w.Header().Set("Content-Type", "text/html") - - data := InitPageData(w, r, "faq", "/faq", "FAQ", templateFiles) - - if handleTemplateError(w, r, "faq.go", "Faq", "", faqTemplate.ExecuteTemplate(w, "layout", data)) != nil { - return // an error has occurred and was processed - } -} diff --git a/handlers/graffitiwall.go b/handlers/graffitiwall.go index 7e861bedc0..6462b2b5b9 100644 --- a/handlers/graffitiwall.go +++ b/handlers/graffitiwall.go @@ -4,6 +4,7 @@ import ( "eth2-exporter/db" "eth2-exporter/templates" "eth2-exporter/types" + "eth2-exporter/utils" "net/http" ) @@ -21,7 +22,7 @@ func Graffitiwall(w http.ResponseWriter, r *http.Request) { err = db.ReaderDb.Select(&graffitiwallData, "SELECT DISTINCT ON (x, y) x, y, color, slot, validator from graffitiwall ORDER BY x, y, slot DESC") if err != nil { - logger.Errorf("error retrieving block tree data: %v", err) + utils.LogError(err, "error retrieving graffitiwall data", 0) http.Error(w, "Internal server error", http.StatusServiceUnavailable) return } diff --git a/handlers/imprint.go b/handlers/imprint.go index 63eca0caf0..efe991934c 100644 --- a/handlers/imprint.go +++ b/handlers/imprint.go @@ -5,51 +5,18 @@ import ( "eth2-exporter/utils" "html/template" "net/http" - "os" - "path" ) // Imprint will show the imprint data using a go template func Imprint(w http.ResponseWriter, r *http.Request) { - templateFiles := []string{getImprintPath()} - if len(templateFiles) == 0 { - templateFiles = append(layoutTemplateFiles, "imprint.example.html") - } + + imprintTemplate := templates.GetTemplate(layoutTemplateFiles...) + imprintTemplate = template.Must(imprintTemplate.Parse(utils.Config.Frontend.Legal.ImprintTemplate)) w.Header().Set("Content-Type", "text/html") - data := InitPageData(w, r, "imprint", "/imprint", "Imprint", templateFiles) + data := InitPageData(w, r, "imprint", "/imprint", "Imprint", layoutTemplateFiles) - if handleTemplateError(w, r, "imprint.go", "Imprint", "", getImprintTemplate(getImprintPath()).ExecuteTemplate(w, "layout", data)) != nil { + if handleTemplateError(w, r, "imprint.go", "Imprint", "", imprintTemplate.ExecuteTemplate(w, "layout", data)) != nil { return // an error has occurred and was processed } } - -func CheckAndPreloadImprint() error { - imprintPath := getImprintPath() - if len(imprintPath) > 0 { - _, err := os.Stat(imprintPath) // check file exists - if err != nil { - return err - } - } - - getImprintTemplate(imprintPath) // preload - return nil -} - -func getImprintPath() string { - if utils.Config.Frontend.LegalDir == "" { - return utils.Config.Frontend.Imprint - } - return path.Join(utils.Config.Frontend.LegalDir, "index.html") -} - -func getImprintTemplate(path string) *template.Template { - if len(path) == 0 { - return templates.GetTemplate(append(layoutTemplateFiles, "imprint.example.html")...) - } - - var imprintTemplate = templates.GetTemplate(layoutTemplateFiles...) - imprintTemplate = templates.AddTemplateFile(imprintTemplate, path) - return imprintTemplate -} diff --git a/handlers/pageData.go b/handlers/pageData.go index c18a53bc09..fd211bd0dd 100644 --- a/handlers/pageData.go +++ b/handlers/pageData.go @@ -89,6 +89,8 @@ func InitPageData(w http.ResponseWriter, r *http.Request, active, path, title st GlobalNotification: services.GlobalNotificationMessage(), AvailableCurrencies: price.GetAvailableCurrencies(), MainMenuItems: createMenuItems(active, isMainnet), + TermsOfServiceUrl: utils.Config.Frontend.Legal.TermsOfServiceUrl, + PrivacyPolicyUrl: utils.Config.Frontend.Legal.PrivacyPolicyUrl, } adConfigurations, err := db.GetAdConfigurationsForTemplate(mainTemplates, data.NoAds) diff --git a/handlers/poap.go b/handlers/poap.go deleted file mode 100644 index 7f58d14494..0000000000 --- a/handlers/poap.go +++ /dev/null @@ -1,158 +0,0 @@ -package handlers - -import ( - "encoding/base64" - "encoding/json" - "eth2-exporter/db" - "eth2-exporter/services" - "eth2-exporter/templates" - "eth2-exporter/types" - "eth2-exporter/utils" - "fmt" - "net/http" - "strconv" - "sync/atomic" - - eth1common "github.com/ethereum/go-ethereum/common" -) - -// do not change existing entries, only append new entries -var poapClients = []string{"Prysm", "Lighthouse", "Teku", "Nimbus", "Lodestar"} -var poapMaxSlot = uint64(300000) - -var poapData atomic.Value -var poapDataEpoch uint64 - -func Poap(w http.ResponseWriter, r *http.Request) { - templateFiles := append(layoutTemplateFiles, "poap.html") - var poapTemplate = templates.GetTemplate(templateFiles...) - - w.Header().Set("Content-Type", "text/html") - - data := InitPageData(w, r, "more", "/poap", "POAP", templateFiles) - data.Data = struct { - PoapClients []string - }{ - PoapClients: poapClients, - } - - if handleTemplateError(w, r, "poap.go", "Poap", "", poapTemplate.ExecuteTemplate(w, "layout", data)) != nil { - return // an error has occurred and was processed - } -} - -func PoapData(w http.ResponseWriter, r *http.Request) { - w.Header().Set("Content-Type", "application/json") - - latestEpoch := services.LatestEpoch() - latestPoapDataEpoch := atomic.LoadUint64(&poapDataEpoch) - latestPoapData := poapData.Load() - - if latestPoapData != nil && (latestEpoch < latestPoapDataEpoch || latestEpoch == 0 || latestEpoch > utils.EpochOfSlot(poapMaxSlot)) { - err := json.NewEncoder(w).Encode(latestPoapData.(*types.DataTableResponse)) - if err != nil { - logger.Errorf("error enconding json response for %v route: %v", r.URL.String(), err) - http.Error(w, "Internal server error", http.StatusServiceUnavailable) - return - } - return - } - - sqlRes := []struct { - Graffiti string - Blockcount uint64 - Validatorcount uint64 - }{} - err := db.ReaderDb.Select(&sqlRes, ` - select - graffiti, - count(*) as blockcount, - count(distinct proposer) as validatorcount - from blocks - where slot <= $1 and graffiti like 'poap%' - group by graffiti`, poapMaxSlot) - if err != nil { - logger.Errorf("error retrieving poap data: %v", err) - http.Error(w, "Internal server error", http.StatusServiceUnavailable) - return - } - - // map[]map[][,] - res := map[string]map[string][]uint64{} - - for _, d := range sqlRes { - eth1Addr, client, err := decodePoapGraffiti(d.Graffiti) - if err != nil { - continue - } - _, exists := res[eth1Addr] - if !exists { - res[eth1Addr] = map[string][]uint64{} - for _, name := range poapClients { - res[eth1Addr][name] = []uint64{0, 0} - } - } - res[eth1Addr][client][0] = d.Blockcount - res[eth1Addr][client][1] = d.Validatorcount - } - - // [, , , , , ..] - tableData := [][]interface{}{} - for eth1Addr, d := range res { - f := []interface{}{eth1common.HexToAddress(eth1Addr).Hex(), uint64(0), uint64(0)} - totalBlocks := uint64(0) - totalValidators := uint64(0) - for _, name := range poapClients { - totalBlocks += d[name][0] - totalValidators += d[name][1] - f = append(f, d[name][0]) - f = append(f, d[name][1]) - } - f[1] = totalBlocks - f[2] = totalValidators - tableData = append(tableData, f) - } - - data := &types.DataTableResponse{ - Draw: 1, - RecordsTotal: 1, - RecordsFiltered: 1, - Data: tableData, - } - - poapData.Store(data) - atomic.StoreUint64(&poapDataEpoch, latestEpoch) - - err = json.NewEncoder(w).Encode(data) - if err != nil { - logger.Errorf("error enconding json response for %v route: %v", r.URL.String(), err) - http.Error(w, "Internal server error", http.StatusServiceUnavailable) - return - } -} - -func decodePoapGraffiti(graffiti string) (eth1Address, client string, err error) { - if len(graffiti) != 32 { - return "", "", fmt.Errorf("invalid graffiti-length") - } - b, err := base64.StdEncoding.DecodeString(graffiti[4:]) - if err != nil { - return "", "", fmt.Errorf("failed decoding base64: %w", err) - } - str := fmt.Sprintf("%x", b) - if len(str) != 42 { - return "", "", fmt.Errorf("invalid length") - } - eth1Address = "0x" + str[:40] - if !utils.IsValidEth1Address(eth1Address) { - return "", "", fmt.Errorf("invalid eth1-address: %v", eth1Address) - } - clientID, err := strconv.ParseInt(str[40:], 16, 64) - if err != nil { - return "", "", fmt.Errorf("invalid clientID: %v: %w", str[40:], err) - } - if clientID < 0 || int64(len(poapClients)) < clientID { - return "", "", fmt.Errorf("invalid clientID: %v", str[40:]) - } - return eth1Address, poapClients[clientID], nil -} diff --git a/handlers/pools_rocketpool.go b/handlers/pools_rocketpool.go index dc174e96d5..e2e2c3797e 100644 --- a/handlers/pools_rocketpool.go +++ b/handlers/pools_rocketpool.go @@ -32,20 +32,20 @@ func PoolsRocketpoolDataMinipools(w http.ResponseWriter, r *http.Request) { q := r.URL.Query() draw, err := strconv.ParseUint(q.Get("draw"), 10, 64) if err != nil { - logger.Errorf("error converting datatables data parameter from string to int: %v", err) - http.Error(w, "Internal server error", http.StatusServiceUnavailable) + logger.Warnf("error converting datatables draw parameter from string to int: %v", err) + http.Error(w, "Error: Missing or invalid parameter draw", http.StatusBadRequest) return } start, err := strconv.ParseUint(q.Get("start"), 10, 64) if err != nil { - logger.Errorf("error converting datatables start parameter from string to int: %v", err) - http.Error(w, "Internal server error", http.StatusServiceUnavailable) + logger.Warnf("error converting datatables start parameter from string to int: %v", err) + http.Error(w, "Error: Missing or invalid parameter start", http.StatusBadRequest) return } length, err := strconv.ParseUint(q.Get("length"), 10, 64) if err != nil { - logger.Errorf("error converting datatables length parameter from string to int: %v", err) - http.Error(w, "Internal server error", http.StatusServiceUnavailable) + logger.Warnf("error converting datatables length parameter from string to int: %v", err) + http.Error(w, "Error: Missing or invalid parameter length", http.StatusBadRequest) return } if length > 100 { @@ -56,6 +56,13 @@ func PoolsRocketpoolDataMinipools(w http.ResponseWriter, r *http.Request) { search = search[:128] } + // Search for invalid postgres strings + if utils.HasProblematicUtfCharacters(search) || strings.HasSuffix(search, "\\") { + logger.Warnf("error converting search %v to valid UTF-8): %v", search, err) + http.Error(w, "Error: Invalid parameter search.", http.StatusBadRequest) + return + } + orderColumn := q.Get("order[0][column]") orderByMap := map[string]string{ "0": "address", @@ -188,20 +195,20 @@ func PoolsRocketpoolDataNodes(w http.ResponseWriter, r *http.Request) { q := r.URL.Query() draw, err := strconv.ParseUint(q.Get("draw"), 10, 64) if err != nil { - logger.Errorf("error converting datatables data parameter from string to int: %v", err) - http.Error(w, "Internal server error", http.StatusServiceUnavailable) + logger.Warnf("error converting datatables draw parameter from string to int: %v", err) + http.Error(w, "Error: Missing or invalid parameter draw", http.StatusBadRequest) return } start, err := strconv.ParseUint(q.Get("start"), 10, 64) if err != nil { - logger.Errorf("error converting datatables start parameter from string to int: %v", err) - http.Error(w, "Internal server error", http.StatusServiceUnavailable) + logger.Warnf("error converting datatables start parameter from string to int: %v", err) + http.Error(w, "Error: Missing or invalid parameter start", http.StatusBadRequest) return } length, err := strconv.ParseUint(q.Get("length"), 10, 64) if err != nil { - logger.Errorf("error converting datatables length parameter from string to int: %v", err) - http.Error(w, "Internal server error", http.StatusServiceUnavailable) + logger.Warnf("error converting datatables length parameter from string to int: %v", err) + http.Error(w, "Error: Missing or invalid parameter length", http.StatusBadRequest) return } if length > 100 { @@ -212,6 +219,13 @@ func PoolsRocketpoolDataNodes(w http.ResponseWriter, r *http.Request) { search = search[:128] } + // Search for invalid postgres strings + if utils.HasProblematicUtfCharacters(search) || strings.HasSuffix(search, "\\") { + logger.Warnf("error converting search %v to valid UTF-8): %v", search, err) + http.Error(w, "Error: Invalid parameter search.", http.StatusBadRequest) + return + } + orderColumn := q.Get("order[0][column]") orderByMap := map[string]string{ "0": "address", @@ -332,20 +346,20 @@ func PoolsRocketpoolDataDAOProposals(w http.ResponseWriter, r *http.Request) { q := r.URL.Query() draw, err := strconv.ParseUint(q.Get("draw"), 10, 64) if err != nil { - logger.Errorf("error converting datatables data parameter from string to int: %v", err) - http.Error(w, "Internal server error", http.StatusServiceUnavailable) + logger.Warnf("error converting datatables draw parameter from string to int: %v", err) + http.Error(w, "Error: Missing or invalid parameter draw", http.StatusBadRequest) return } start, err := strconv.ParseUint(q.Get("start"), 10, 64) if err != nil { - logger.Errorf("error converting datatables start parameter from string to int: %v", err) - http.Error(w, "Internal server error", http.StatusServiceUnavailable) + logger.Warnf("error converting datatables start parameter from string to int: %v", err) + http.Error(w, "Error: Missing or invalid parameter start", http.StatusBadRequest) return } length, err := strconv.ParseUint(q.Get("length"), 10, 64) if err != nil { - logger.Errorf("error converting datatables length parameter from string to int: %v", err) - http.Error(w, "Internal server error", http.StatusServiceUnavailable) + logger.Warnf("error converting datatables length parameter from string to int: %v", err) + http.Error(w, "Error: Missing or invalid parameter length", http.StatusBadRequest) return } if length > 100 { @@ -356,6 +370,13 @@ func PoolsRocketpoolDataDAOProposals(w http.ResponseWriter, r *http.Request) { search = search[:128] } + // Search for invalid postgres strings + if utils.HasProblematicUtfCharacters(search) || strings.HasSuffix(search, "\\") { + logger.Warnf("error converting search %v to valid UTF-8): %v", search, err) + http.Error(w, "Error: Invalid parameter search.", http.StatusBadRequest) + return + } + orderColumn := q.Get("order[0][column]") orderByMap := map[string]string{ "0": "id", @@ -545,20 +566,20 @@ func PoolsRocketpoolDataDAOMembers(w http.ResponseWriter, r *http.Request) { q := r.URL.Query() draw, err := strconv.ParseUint(q.Get("draw"), 10, 64) if err != nil { - logger.Errorf("error converting datatables data parameter from string to int: %v", err) - http.Error(w, "Internal server error", http.StatusServiceUnavailable) + logger.Warnf("error converting datatables draw parameter from string to int: %v", err) + http.Error(w, "Error: Missing or invalid parameter draw", http.StatusBadRequest) return } start, err := strconv.ParseUint(q.Get("start"), 10, 64) if err != nil { - logger.Errorf("error converting datatables start parameter from string to int: %v", err) - http.Error(w, "Internal server error", http.StatusServiceUnavailable) + logger.Warnf("error converting datatables start parameter from string to int: %v", err) + http.Error(w, "Error: Missing or invalid parameter start", http.StatusBadRequest) return } length, err := strconv.ParseUint(q.Get("length"), 10, 64) if err != nil { - logger.Errorf("error converting datatables length parameter from string to int: %v", err) - http.Error(w, "Internal server error", http.StatusServiceUnavailable) + logger.Warnf("error converting datatables length parameter from string to int: %v", err) + http.Error(w, "Error: Missing or invalid parameter length", http.StatusBadRequest) return } if length > 100 { @@ -569,6 +590,13 @@ func PoolsRocketpoolDataDAOMembers(w http.ResponseWriter, r *http.Request) { search = search[:128] } + // Search for invalid postgres strings + if utils.HasProblematicUtfCharacters(search) || strings.HasSuffix(search, "\\") { + logger.Warnf("error converting search %v to valid UTF-8): %v", search, err) + http.Error(w, "Error: Invalid parameter search.", http.StatusBadRequest) + return + } + orderColumn := q.Get("order[0][column]") orderByMap := map[string]string{ "0": "address", diff --git a/handlers/search.go b/handlers/search.go index b2c92a3a82..a601e6b943 100644 --- a/handlers/search.go +++ b/handlers/search.go @@ -69,14 +69,21 @@ func SearchAhead(w http.ResponseWriter, r *http.Request) { vars := mux.Vars(r) searchType := vars["type"] search := vars["search"] - search = strings.Replace(search, "0x", "", -1) var err error logger := logger.WithField("searchType", searchType) var result interface{} + strippedSearch := strings.Replace(search, "0x", "", -1) + lowerStrippedSearch := strings.ToLower(strippedSearch) + + if len(strippedSearch) == 0 { + _ = json.NewEncoder(w).Encode(result) + return + } + switch searchType { case "slots": - if len(search) <= 1 || !searchLikeRE.MatchString(search) { + if len(search) <= 1 || !searchLikeRE.MatchString(strippedSearch) { break } result = &types.SearchAheadSlotsResult{} @@ -86,9 +93,9 @@ func SearchAhead(w http.ResponseWriter, r *http.Request) { FROM blocks WHERE slot = $1 ORDER BY slot LIMIT 10`, search) - } else if len(search) == 64 { + } else if len(strippedSearch) == 64 { var blockHash []byte - blockHash, err = hex.DecodeString(search) + blockHash, err = hex.DecodeString(strippedSearch) if err != nil { err = fmt.Errorf("error parsing blockHash to int: %v", err) break @@ -133,13 +140,12 @@ func SearchAhead(w http.ResponseWriter, r *http.Request) { } result = graffiti case "transactions": - search = strings.ToLower(strings.Replace(search, "0x", "", -1)) - if !transactionLikeRE.MatchString(search) { + if !transactionLikeRE.MatchString(lowerStrippedSearch) { break } result = &types.SearchAheadTransactionsResult{} var txHash []byte - txHash, err = hex.DecodeString(search) + txHash, err = hex.DecodeString(lowerStrippedSearch) if err != nil { err = fmt.Errorf("error parsing txHash %v: %v", search, err) break @@ -163,8 +169,8 @@ func SearchAhead(w http.ResponseWriter, r *http.Request) { indexNumeric, parseErr := strconv.ParseInt(search, 10, 32) if parseErr == nil { // search the validator by its index err = db.ReaderDb.Select(result, `SELECT validatorindex AS index, pubkeyhex as pubkey FROM validators WHERE validatorindex = $1`, indexNumeric) - } else if thresholdHexLikeRE.MatchString(search) { - err = db.ReaderDb.Select(result, `SELECT validatorindex AS index, pubkeyhex as pubkey FROM validators WHERE pubkeyhex LIKE LOWER($1 || '%')`, search) + } else if thresholdHexLikeRE.MatchString(lowerStrippedSearch) { + err = db.ReaderDb.Select(result, `SELECT validatorindex AS index, pubkeyhex as pubkey FROM validators WHERE pubkeyhex LIKE ($1 || '%')`, lowerStrippedSearch) } else { err = db.ReaderDb.Select(result, ` SELECT validatorindex AS index, pubkeyhex AS pubkey @@ -185,13 +191,13 @@ func SearchAhead(w http.ResponseWriter, r *http.Request) { break } } - if !searchLikeRE.MatchString(search) { + if !searchLikeRE.MatchString(strippedSearch) { break } - if len(search)%2 != 0 { // pad with 0 if uneven - search = search + "0" + if len(strippedSearch)%2 != 0 { // pad with 0 if uneven + strippedSearch = strippedSearch + "0" } - eth1AddressHash, decodeErr := hex.DecodeString(search) + eth1AddressHash, decodeErr := hex.DecodeString(strippedSearch) if decodeErr != nil { break } @@ -200,13 +206,14 @@ func SearchAhead(w http.ResponseWriter, r *http.Request) { err = fmt.Errorf("error searching for eth1AddressHash: %v", err) } case "indexed_validators": + // find all validators that have a publickey or index like the search-query result = &types.SearchAheadValidatorsResult{} indexNumeric, errParse := strconv.ParseInt(search, 10, 32) if errParse == nil { // search the validator by its index err = db.ReaderDb.Select(result, `SELECT validatorindex AS index, pubkeyhex as pubkey FROM validators WHERE validatorindex = $1`, indexNumeric) - } else if thresholdHexLikeRE.MatchString(search) { - err = db.ReaderDb.Select(result, `SELECT validatorindex AS index, pubkeyhex as pubkey FROM validators WHERE pubkeyhex LIKE LOWER($1 || '%')`, search) + } else if thresholdHexLikeRE.MatchString(lowerStrippedSearch) { + err = db.ReaderDb.Select(result, `SELECT validatorindex AS index, pubkeyhex as pubkey FROM validators WHERE pubkeyhex LIKE ($1 || '%')`, lowerStrippedSearch) } else { err = db.ReaderDb.Select(result, ` SELECT validatorindex AS index, pubkeyhex AS pubkey @@ -216,7 +223,7 @@ func SearchAhead(w http.ResponseWriter, r *http.Request) { ORDER BY index LIMIT 10`, search+"%") } case "validators_by_pubkey": - if !thresholdHexLikeRE.MatchString(search) { + if !thresholdHexLikeRE.MatchString(lowerStrippedSearch) { break } result = &types.SearchAheadPubkeyResult{} @@ -226,9 +233,10 @@ func SearchAhead(w http.ResponseWriter, r *http.Request) { ENCODE(eth1_deposits.publickey, 'hex') AS pubkey FROM eth1_deposits LEFT JOIN validators ON validators.pubkey = eth1_deposits.publickey - WHERE validators.pubkey IS NULL AND ENCODE(eth1_deposits.publickey, 'hex') LIKE ($1 || '%')`, search) + WHERE validators.pubkey IS NULL AND ENCODE(eth1_deposits.publickey, 'hex') LIKE ($1 || '%')`, lowerStrippedSearch) case "indexed_validators_by_eth1_addresses": - if !utils.IsValidEnsDomain(search) && !utils.IsEth1Address(search) { + search = ReplaceEnsNameWithAddress(search) + if !utils.IsEth1Address(search) { break } result, err = FindValidatorIndicesByEth1Address(strings.ToLower(search)) @@ -237,24 +245,19 @@ func SearchAhead(w http.ResponseWriter, r *http.Request) { if utils.IsValidEnsDomain(search) { ensData, _ = GetEnsDomain(search) if len(ensData.Address) > 0 { - search = strings.Replace(ensData.Address, "0x", "", -1) + lowerStrippedSearch = strings.ToLower(strings.Replace(ensData.Address, "0x", "", -1)) } } - if !searchLikeRE.MatchString(search) { + if !searchLikeRE.MatchString(lowerStrippedSearch) { break } - if len(search)%2 != 0 { // pad with 0 if uneven - search = search + "0" - } // find validators per eth1-address (limit result by N addresses and M validators per address) result = &[]struct { Eth1Address string `db:"from_address_text" json:"eth1_address"` Count uint64 `db:"count" json:"count"` }{} - if searchLikeRE.MatchString(search) { - trimmed := strings.ToLower(strings.TrimPrefix(search, "0x")) - err = db.ReaderDb.Select(result, ` + err = db.ReaderDb.Select(result, ` SELECT from_address_text, COUNT(*) FROM ( SELECT DISTINCT ON(validatorindex) validatorindex, @@ -263,8 +266,8 @@ func SearchAhead(w http.ResponseWriter, r *http.Request) { INNER JOIN validators ON validators.pubkey = eth1_deposits.publickey WHERE from_address_text LIKE $1 || '%' ) a - GROUP BY from_address_text`, trimmed) - } + GROUP BY from_address_text`, lowerStrippedSearch) + case "indexed_validators_by_graffiti": // find validators per graffiti (limit result by N graffities and M validators per graffiti) res := []struct { @@ -352,57 +355,32 @@ func SearchAhead(w http.ResponseWriter, r *http.Request) { // search can ether be a valid ETH address or an ENS name mapping to one func FindValidatorIndicesByEth1Address(search string) (types.SearchValidatorsByEth1Result, error) { + search = strings.ToLower(strings.Replace(ReplaceEnsNameWithAddress(search), "0x", "", -1)) + if !utils.IsValidEth1Address(search) { + return nil, fmt.Errorf("not a valid Eth1 address: %v", search) + } + // find validators per eth1-address (limit result by N addresses and M validators per address) + result := &[]struct { Eth1Address string `db:"from_address_text" json:"eth1_address"` ValidatorIndices pq.Int64Array `db:"validatorindices" json:"validator_indices"` Count uint64 `db:"count" json:"-"` }{} - - search = strings.Replace(ReplaceEnsNameWithAddress(search), "0x", "", -1) - if len(search)%2 != 0 { - search = search[:len(search)-1] - } - if len(search) <= 1 || len(search) > 40 { - return nil, fmt.Errorf("not a valid Eth1 address: %v", search) - } - // find validators per eth1-address (limit result by N addresses and M validators per address) - - eth1AddressHash, err := hex.DecodeString(search) - if err != nil { - return nil, fmt.Errorf("error parsing eth1AddressHash to hex: %v", err) - } - if len(eth1AddressHash) == 20 { - // if it is an eth1-address just search for exact match - err = db.ReaderDb.Select(result, ` - SELECT from_address_text, COUNT(*), ARRAY_AGG(validatorindex) validatorindices FROM ( - SELECT - DISTINCT ON(validatorindex) validatorindex, - from_address_text, - DENSE_RANK() OVER (PARTITION BY from_address_text ORDER BY validatorindex) AS validatorrow, - DENSE_RANK() OVER (ORDER BY from_address_text) AS addressrow - FROM eth1_deposits - INNER JOIN validators ON validators.pubkey = eth1_deposits.publickey - WHERE from_address_text = $1 - ) a - WHERE validatorrow <= $2 AND addressrow <= 10 - GROUP BY from_address_text - ORDER BY count DESC`, search, searchValidatorsResultLimit) - } else { - err = db.ReaderDb.Select(result, ` - SELECT from_address_text, COUNT(*), ARRAY_AGG(validatorindex) validatorindices FROM ( - SELECT - DISTINCT ON(validatorindex) validatorindex, - from_address_text, - DENSE_RANK() OVER (PARTITION BY from_address_text ORDER BY validatorindex) AS validatorrow, - DENSE_RANK() OVER (ORDER BY from_address_text) AS addressrow - FROM eth1_deposits - INNER JOIN validators ON validators.pubkey = eth1_deposits.publickey - WHERE from_address_text LIKE $1 || '%' - ) a - WHERE validatorrow <= $2 AND addressrow <= 10 - GROUP BY from_address_text - ORDER BY count DESC`, search, searchValidatorsResultLimit) - } + // just search for exact match (substring matches turned out to be too heavy for the db!) + err := db.ReaderDb.Select(result, ` + SELECT from_address_text, COUNT(*), ARRAY_AGG(validatorindex) validatorindices FROM ( + SELECT + DISTINCT ON(validatorindex) validatorindex, + from_address_text, + DENSE_RANK() OVER (PARTITION BY from_address_text ORDER BY validatorindex) AS validatorrow, + DENSE_RANK() OVER (ORDER BY from_address_text) AS addressrow + FROM eth1_deposits + INNER JOIN validators ON validators.pubkey = eth1_deposits.publickey + WHERE from_address_text = $1 + ) a + WHERE validatorrow <= $2 AND addressrow <= 10 + GROUP BY from_address_text + ORDER BY count DESC`, search, searchValidatorsResultLimit) if err != nil { utils.LogError(err, "error getting validators for eth1 address from db", 0) return nil, fmt.Errorf("error reading result data: %v", err) diff --git a/handlers/slot.go b/handlers/slot.go index 4472ebebba..f74516d08d 100644 --- a/handlers/slot.go +++ b/handlers/slot.go @@ -5,11 +5,13 @@ import ( "encoding/hex" "encoding/json" "eth2-exporter/db" + "eth2-exporter/services" "eth2-exporter/templates" "eth2-exporter/types" "eth2-exporter/utils" "fmt" "html/template" + "math" "math/big" "net/http" "strconv" @@ -61,7 +63,7 @@ func Slot(w http.ResponseWriter, r *http.Request) { if err != nil || len(slotOrHash) != 64 { blockRootHash = []byte{} blockSlot, err = strconv.ParseInt(vars["slotOrHash"], 10, 64) - if err != nil || blockSlot >= 2147483648 { // block slot must be lower then max int4 + if err != nil || blockSlot > math.MaxInt32 { // block slot must be lower than max int4 data := InitPageData(w, r, "blockchain", "/slots", fmt.Sprintf("Slot %v", slotOrHash), blockNotFoundTemplateFiles) data.Data = "slot" if handleTemplateError(w, r, "slot.go", "Slot", "blockSlot", blockNotFoundTemplate.ExecuteTemplate(w, "layout", data)) != nil { @@ -206,14 +208,15 @@ func getAttestationsData(slot uint64, onlyFirst bool) ([]*types.BlockPageAttesta } func GetSlotPageData(blockSlot uint64) (*types.BlockPageData, error) { + latestFinalizedEpoch := services.LatestFinalizedEpoch() blockPageData := types.BlockPageData{} blockPageData.Mainnet = utils.Config.Chain.ClConfig.ConfigName == "mainnet" // for the first slot in an epoch the previous epoch defines the finalized state err := db.ReaderDb.Get(&blockPageData, ` SELECT blocks.epoch, - COALESCE(epochs.finalized, false) AS epoch_finalized, - COALESCE(prev_epoch.finalized, false) AS prev_epoch_finalized, + (COALESCE(epochs.epoch, 0) <= $3) AS epoch_finalized, + (GREATEST((blocks.slot-1)/$2-1,0) <= $3) AS prev_epoch_finalized, COALESCE(epochs.globalparticipationrate, 0) AS epoch_participation_rate, blocks.slot, blocks.blockroot, @@ -247,7 +250,6 @@ func GetSlotPageData(blockSlot uint64) (*types.BlockPageData, error) { LEFT JOIN blocks_tags ON blocks.slot = blocks_tags.slot and blocks.blockroot = blocks_tags.blockroot LEFT JOIN tags ON blocks_tags.tag_id = tags.id LEFT JOIN epochs ON GREATEST((blocks.slot-1)/$2,0) = epochs.epoch - LEFT JOIN epochs prev_epoch ON GREATEST((blocks.slot-1)/$2-1,0) = prev_epoch.epoch WHERE blocks.slot = $1 group by blocks.epoch, @@ -255,11 +257,10 @@ func GetSlotPageData(blockSlot uint64) (*types.BlockPageData, error) { blocks.blockroot, validator_names."name", epoch_finalized, - prev_epoch.finalized, epoch_participation_rate ORDER BY blocks.blockroot DESC, blocks.status ASC limit 1 `, - blockSlot, utils.Config.Chain.ClConfig.SlotsPerEpoch) + blockSlot, utils.Config.Chain.ClConfig.SlotsPerEpoch, latestFinalizedEpoch) if err != nil { return nil, err } @@ -382,8 +383,8 @@ func SlotDepositData(w http.ResponseWriter, r *http.Request) { if err != nil || len(slotOrHash) != 64 { blockSlot, err = strconv.ParseInt(vars["slotOrHash"], 10, 64) if err != nil { - logger.Errorf("error parsing slotOrHash url parameter %v, err: %v", vars["slotOrHash"], err) - http.Error(w, "Internal server error", http.StatusInternalServerError) + logger.Warnf("error parsing slotOrHash url parameter %v, err: %v", vars["slotOrHash"], err) + http.Error(w, "Error: Invalid parameter slotOrHash.", http.StatusBadRequest) return } } else { @@ -404,21 +405,21 @@ func SlotDepositData(w http.ResponseWriter, r *http.Request) { draw, err := strconv.ParseUint(q.Get("draw"), 10, 64) if err != nil { - logger.Errorf("error converting datatables data parameter from string to int: %v", err) - http.Error(w, "Internal server error", http.StatusInternalServerError) + logger.Warnf("error converting datatables draw parameter from string to int: %v", err) + http.Error(w, "Error: Missing or invalid parameter draw", http.StatusBadRequest) return } start, err := strconv.ParseUint(q.Get("start"), 10, 64) if err != nil { - logger.Errorf("error converting datatables start parameter from string to int: %v", err) - http.Error(w, "Internal server error", http.StatusInternalServerError) + logger.Warnf("error converting datatables start parameter from string to int: %v", err) + http.Error(w, "Error: Missing or invalid parameter start", http.StatusBadRequest) return } length, err := strconv.ParseUint(q.Get("length"), 10, 64) if err != nil { - logger.Errorf("error converting datatables length parameter from string to int: %v", err) - http.Error(w, "Internal server error", http.StatusInternalServerError) + logger.Warnf("error converting datatables length parameter from string to int: %v", err) + http.Error(w, "Error: Missing or invalid parameter length", http.StatusBadRequest) return } @@ -504,8 +505,8 @@ func SlotVoteData(w http.ResponseWriter, r *http.Request) { if err != nil || len(slotOrHash) != 64 { blockSlot, err = strconv.ParseInt(vars["slotOrHash"], 10, 64) if err != nil { - logger.Errorf("error parsing slotOrHash url parameter %v, err: %v", vars["slotOrHash"], err) - http.Error(w, "Internal server error", http.StatusInternalServerError) + logger.Warnf("error parsing slotOrHash url parameter %v, err: %v", vars["slotOrHash"], err) + http.Error(w, "Error: Invalid parameter slotOrHash.", http.StatusBadRequest) return } err = db.ReaderDb.Get(&blockRootHash, "select blocks.blockroot from blocks where blocks.slot = $1", blockSlot) @@ -526,29 +527,29 @@ func SlotVoteData(w http.ResponseWriter, r *http.Request) { q := r.URL.Query() search := q.Get("search[value]") - searchIsUint64 := false - searchUint64, err := strconv.ParseUint(search, 10, 64) - if err == nil { - searchIsUint64 = true + searchIsInt32 := false + searchInt32, err := strconv.ParseInt(search, 10, 32) + if err == nil && searchInt32 >= 0 { + searchIsInt32 = true } draw, err := strconv.ParseUint(q.Get("draw"), 10, 64) if err != nil { - logger.Errorf("error converting datatables data parameter from string to int: %v", err) - http.Error(w, "Internal server error", http.StatusInternalServerError) + logger.Warnf("error converting datatables draw parameter from string to int: %v", err) + http.Error(w, "Error: Missing or invalid parameter draw", http.StatusBadRequest) return } start, err := strconv.ParseUint(q.Get("start"), 10, 64) if err != nil { - logger.Errorf("error converting datatables start parameter from string to int: %v", err) - http.Error(w, "Internal server error", http.StatusInternalServerError) + logger.Warnf("error converting datatables start parameter from string to int: %v", err) + http.Error(w, "Error: Missing or invalid parameter start", http.StatusBadRequest) return } length, err := strconv.ParseUint(q.Get("length"), 10, 64) if err != nil { - logger.Errorf("error converting datatables length parameter from string to int: %v", err) - http.Error(w, "Internal server error", http.StatusInternalServerError) + logger.Warnf("error converting datatables length parameter from string to int: %v", err) + http.Error(w, "Error: Missing or invalid parameter length", http.StatusBadRequest) return } @@ -587,8 +588,8 @@ func SlotVoteData(w http.ResponseWriter, r *http.Request) { http.Error(w, "Internal server error", http.StatusInternalServerError) return } - } else if searchIsUint64 { - err = db.ReaderDb.Get(&count, `SELECT count(*) FROM blocks_attestations WHERE beaconblockroot = $1 AND $2 = ANY(validators)`, blockRootHash, searchUint64) + } else if searchIsInt32 { + err = db.ReaderDb.Get(&count, `SELECT count(*) FROM blocks_attestations WHERE beaconblockroot = $1 AND $2 = ANY(validators)`, blockRootHash, searchInt32) if err != nil { logger.Errorf("error retrieving deposit count for slot %v", err) http.Error(w, "Internal server error", http.StatusInternalServerError) @@ -605,7 +606,7 @@ func SlotVoteData(w http.ResponseWriter, r *http.Request) { ORDER BY committeeindex LIMIT $3 OFFSET $4`, - blockRootHash, searchUint64, length, start) + blockRootHash, searchInt32, length, start) if err != nil { logger.Errorf("error retrieving block vote data: %v", err) http.Error(w, "Internal server error", http.StatusInternalServerError) @@ -661,8 +662,8 @@ func BlockTransactionsData(w http.ResponseWriter, r *http.Request) { vars := mux.Vars(r) slot, err := strconv.ParseUint(vars["block"], 10, 64) if err != nil { - logger.Errorf("error parsing slot url parameter %v, err: %v", vars["slot"], err) - http.Error(w, "Internal server error", http.StatusInternalServerError) + logger.Warnf("error parsing slot url parameter %v: %v", vars["slot"], err) + http.Error(w, "Error: Invalid parameter slot.", http.StatusBadRequest) return } @@ -718,9 +719,9 @@ func SlotAttestationsData(w http.ResponseWriter, r *http.Request) { vars := mux.Vars(r) slot, err := strconv.ParseUint(vars["slot"], 10, 64) - if err != nil { - logger.Errorf("error parsing slot url parameter %v, err: %v", vars["slot"], err) - http.Error(w, "Internal server error", http.StatusInternalServerError) + if err != nil || slot > math.MaxInt32 { + logger.Warnf("error parsing slot url parameter %v: %v", vars["slot"], err) + http.Error(w, "Error: Invalid parameter slot.", http.StatusBadRequest) return } @@ -765,9 +766,9 @@ func SlotWithdrawalData(w http.ResponseWriter, r *http.Request) { vars := mux.Vars(r) slot, err := strconv.ParseUint(vars["slot"], 10, 64) - if err != nil { - logger.Errorf("error parsing slot url parameter %v, err: %v", vars["slot"], err) - http.Error(w, "Internal server error", http.StatusInternalServerError) + if err != nil || slot > math.MaxInt32 { + logger.Warnf("error parsing slot url parameter %v: %v", vars["slot"], err) + http.Error(w, "Error: Invalid parameter slot.", http.StatusBadRequest) return } withdrawals, err := db.GetSlotWithdrawals(slot) @@ -806,9 +807,9 @@ func SlotBlsChangeData(w http.ResponseWriter, r *http.Request) { vars := mux.Vars(r) slot, err := strconv.ParseUint(vars["slot"], 10, 64) - if err != nil { - logger.Errorf("error parsing slot url parameter %v, err: %v", vars["slot"], err) - http.Error(w, "Internal server error", http.StatusInternalServerError) + if err != nil || slot > math.MaxInt32 { + logger.Warnf("error parsing slot url parameter %v: %v", vars["slot"], err) + http.Error(w, "Error: Invalid parameter slot.", http.StatusBadRequest) return } blsChange, err := db.GetSlotBLSChange(slot) diff --git a/handlers/slots.go b/handlers/slots.go index 070cfb1ab3..613c178980 100644 --- a/handlers/slots.go +++ b/handlers/slots.go @@ -47,20 +47,20 @@ func SlotsData(w http.ResponseWriter, r *http.Request) { draw, err := strconv.ParseUint(q.Get("draw"), 10, 64) if err != nil { - logger.Errorf("error converting datatables data parameter from string to int: %v", err) - http.Error(w, "Internal server error", http.StatusServiceUnavailable) + logger.Warnf("error converting datatables draw parameter from string to int: %v", err) + http.Error(w, "Error: Missing or invalid parameter draw", http.StatusBadRequest) return } start, err := strconv.ParseUint(q.Get("start"), 10, 64) if err != nil { - logger.Errorf("error converting datatables start parameter from string to int: %v", err) - http.Error(w, "Internal server error", http.StatusServiceUnavailable) + logger.Warnf("error converting datatables start parameter from string to int: %v", err) + http.Error(w, "Error: Missing or invalid parameter start", http.StatusBadRequest) return } length, err := strconv.ParseUint(q.Get("length"), 10, 64) if err != nil { - logger.Errorf("error converting datatables length parameter from string to int: %v", err) - http.Error(w, "Internal server error", http.StatusServiceUnavailable) + logger.Warnf("error converting datatables length parameter from string to int: %v", err) + http.Error(w, "Error: Missing or invalid parameter length", http.StatusBadRequest) return } diff --git a/handlers/stripe.go b/handlers/stripe.go index dafd6bf9db..af11dd67e4 100644 --- a/handlers/stripe.go +++ b/handlers/stripe.go @@ -11,7 +11,6 @@ import ( "fmt" "html/template" "io" - "io/ioutil" "net/http" "time" @@ -195,7 +194,7 @@ func StripeWebhook(w http.ResponseWriter, r *http.Request) { http.Error(w, http.StatusText(http.StatusMethodNotAllowed), http.StatusMethodNotAllowed) return } - b, err := ioutil.ReadAll(r.Body) + b, err := io.ReadAll(r.Body) if err != nil { http.Error(w, err.Error(), http.StatusBadRequest) logger.WithError(err).Error("error failed to read body for StripeWebhook") diff --git a/handlers/unitConverter.go b/handlers/unitConverter.go index eaf37827be..9dee44b099 100644 --- a/handlers/unitConverter.go +++ b/handlers/unitConverter.go @@ -5,7 +5,7 @@ import ( "net/http" ) -// Faq will return the data from the frequently asked questions (FAQ) using a go template +// UnitConverter renders unitConverter template func UnitConverter(w http.ResponseWriter, r *http.Request) { templateFiles := append(layoutTemplateFiles, "unitConverter.html") var unitConverterTemplate = templates.GetTemplate(templateFiles...) diff --git a/handlers/user.go b/handlers/user.go index b88d749671..b946175164 100644 --- a/handlers/user.go +++ b/handlers/user.go @@ -14,7 +14,6 @@ import ( "fmt" "html/template" "io" - "io/ioutil" "net/http" "net/url" "strconv" @@ -181,12 +180,20 @@ func UserAuthorizeConfirm(w http.ResponseWriter, r *http.Request) { clientID := q.Get("client_id") state := q.Get("state") - session.SetValue("state", state) session.SetValue("client_id", clientID) - session.SetValue("oauth_redirect_uri", redirectURI) session.Save(r, w) if !user.Authenticated { + if redirectURI != "" { + var stateParam = "" + if state != "" { + stateParam = "&state=" + state + } + + http.Redirect(w, r, "/login?redirect_uri="+redirectURI+stateParam, http.StatusSeeOther) + return + } + http.Redirect(w, r, "/login", http.StatusSeeOther) return } @@ -223,21 +230,6 @@ func UserAuthorizeConfirm(w http.ResponseWriter, r *http.Request) { } } -// UserAuthorizationCancel cancels oauth authorization session states and redirects to frontpage -func UserAuthorizationCancel(w http.ResponseWriter, r *http.Request) { - _, session, err := getUserSession(r) - if err != nil { - http.Redirect(w, r, "/", http.StatusSeeOther) - return - } - - session.DeleteValue("oauth_redirect_uri") - session.DeleteValue("state") - session.Save(r, w) - - http.Redirect(w, r, "/", http.StatusSeeOther) -} - func UserNotifications(w http.ResponseWriter, r *http.Request) { templateFiles := append(layoutTemplateFiles, "user/notifications.html") var notificationTemplate = templates.GetTemplate(templateFiles...) @@ -352,7 +344,7 @@ func RemoveAllValidatorsAndUnsubscribe(w http.ResponseWriter, r *http.Request) { user := getUser(r) - body, err := ioutil.ReadAll(r.Body) + body, err := io.ReadAll(r.Body) if err != nil { logger.Errorf("error reading body of request: %v, %v", r.URL.String(), err) ErrorOrJSONResponse(w, r, "Internal server error", http.StatusInternalServerError) @@ -382,7 +374,7 @@ func AddValidatorsAndSubscribe(w http.ResponseWriter, r *http.Request) { user := getUser(r) - body, err := ioutil.ReadAll(r.Body) + body, err := io.ReadAll(r.Body) if err != nil { logger.Errorf("error reading body of request: %v, %v", r.URL.String(), err) ErrorOrJSONResponse(w, r, "Internal server error", http.StatusInternalServerError) @@ -454,7 +446,7 @@ func UserUpdateSubscriptions(w http.ResponseWriter, r *http.Request) { user := getUser(r) - body, err := ioutil.ReadAll(r.Body) + body, err := io.ReadAll(r.Body) if err != nil { logger.Errorf("error reading body of request: %v, %v", r.URL.String(), err) ErrorOrJSONResponse(w, r, "Internal server error", http.StatusInternalServerError) @@ -907,25 +899,10 @@ func UserNotificationsData(w http.ResponseWriter, r *http.Request) { draw, err := strconv.ParseUint(q.Get("draw"), 10, 64) if err != nil { - logger.Errorf("error converting datatables data parameter from string to int: %v", err) - http.Error(w, "Internal server error", http.StatusServiceUnavailable) + logger.Warnf("error converting datatables draw parameter from string to int: %v", err) + http.Error(w, "Error: Missing or invalid parameter draw", http.StatusBadRequest) return } - // start, err := strconv.ParseUint(q.Get("start"), 10, 64) - // if err != nil { - // logger.Errorf("error converting datatables start parameter from string to int: %v", err) - // http.Error(w, "Internal server error", http.StatusServiceUnavailable) - // return - // } - // length, err := strconv.ParseUint(q.Get("length"), 10, 64) - // if err != nil { - // logger.Errorf("error converting datatables length parameter from string to int: %v", err) - // http.Error(w, "Internal server error", http.StatusServiceUnavailable) - // return - // } - // if length > 100 { - // length = 100 - // } user := getUser(r) @@ -1020,20 +997,15 @@ func UserSubscriptionsData(w http.ResponseWriter, r *http.Request) { draw, err := strconv.ParseUint(q.Get("draw"), 10, 64) if err != nil { - logger.Errorf("error converting datatables data parameter from string to int: %v", err) - http.Error(w, "Internal server error", http.StatusServiceUnavailable) + logger.Warnf("error converting datatables draw parameter from string to int: %v", err) + http.Error(w, "Error: Missing or invalid parameter draw", http.StatusBadRequest) return } - // start, err := strconv.ParseUint(q.Get("start"), 10, 64) - // if err != nil { - // logger.Errorf("error converting datatables start parameter from string to int: %v", err) - // http.Error(w, "Internal server error", http.StatusServiceUnavailable) - // return - // } + length, err := strconv.ParseUint(q.Get("length"), 10, 64) if err != nil { - logger.Errorf("error converting datatables length parameter from string to int: %v", err) - http.Error(w, "Internal server error", http.StatusServiceUnavailable) + logger.Warnf("error converting datatables length parameter from string to int: %v", err) + http.Error(w, "Error: Missing or invalid parameter length", http.StatusBadRequest) return } if length > 100 { @@ -1181,6 +1153,13 @@ func UserDeletePost(w http.ResponseWriter, r *http.Request) { } Logout(w, r) + err = purgeAllSessionsForUser(r.Context(), user.UserID) + if err != nil { + utils.LogError(err, "error purging sessions for user", 0, map[string]interface{}{"userID": user.UserID}) + utils.SetFlash(w, r, authSessionName, authInternalServerErrorFlashMsg) + http.Redirect(w, r, "/login", http.StatusSeeOther) + return + } } else { utils.LogError(nil, "Trying to delete an unauthenticated user", 0) http.Redirect(w, r, "/user/settings", http.StatusSeeOther) @@ -1620,7 +1599,7 @@ func UserDashboardWatchlistAdd(w http.ResponseWriter, r *http.Request) { SetAutoContentType(w, r) //w.Header().Set("Content-Type", "text/html") user := getUser(r) - body, err := ioutil.ReadAll(r.Body) + body, err := io.ReadAll(r.Body) if err != nil { logger.Errorf("error reading body of request: %v, %v", r.URL.String(), err) ErrorOrJSONResponse(w, r, "Internal server error", http.StatusInternalServerError) @@ -1790,7 +1769,7 @@ func MultipleUsersNotificationsSubscribeWeb(w http.ResponseWriter, r *http.Reque } var jsonObjects []SubIntent - b, err := ioutil.ReadAll(r.Body) + b, err := io.ReadAll(r.Body) if err != nil { logger.Errorf("error reading body %v URL: %v", err, r.URL.String()) sendErrorResponse(w, r.URL.String(), "could not parse body") @@ -1846,11 +1825,8 @@ func internUserNotificationsSubscribe(event, filter string, threshold float64, w return false } - isPkey := searchPubkeyExactRE.MatchString(filter) - filterLen := len(filter) - - if filterLen != 0 && !isPkey { - errMsg := fmt.Errorf("error invalid pubkey characters or length") + if !isValidSubscriptionFilter(eventName, filter) { + errMsg := fmt.Errorf("error invalid filter, not pubkey or client") errFields := map[string]interface{}{ "filter": filter, "filter_len": len(filter)} @@ -1881,6 +1857,7 @@ func internUserNotificationsSubscribe(event, filter string, threshold float64, w // rocketpool thresholds are free } + filterLen := len(filter) if filterLen == 0 && !strings.HasPrefix(string(eventName), "monitoring_") && !strings.HasPrefix(string(eventName), "rocketpool_") { // no filter = add all my watched validators myValidators, err2 := db.GetTaggedValidators(filterWatchlist) if err2 != nil { @@ -2025,11 +2002,8 @@ func internUserNotificationsUnsubscribe(event, filter string, w http.ResponseWri return false } - isPkey := searchPubkeyExactRE.MatchString(filter) - filterLen := len(filter) - - if filterLen != 0 && !isPkey { - errMsg := fmt.Errorf("error invalid pubkey characters or length") + if !isValidSubscriptionFilter(eventName, filter) { + errMsg := fmt.Errorf("error invalid filter, not pubkey or client") errFields := map[string]interface{}{ "filter": filter, "filter_len": len(filter)} @@ -2046,6 +2020,7 @@ func internUserNotificationsUnsubscribe(event, filter string, w http.ResponseWri Network: utils.GetNetwork(), } + filterLen := len(filter) if filterLen == 0 && !strings.HasPrefix(string(eventName), "monitoring_") && !strings.HasPrefix(string(eventName), "rocketpool_") { // no filter = add all my watched validators myValidators, err2 := db.GetTaggedValidators(filterWatchlist) @@ -2113,11 +2088,8 @@ func UserNotificationsUnsubscribe(w http.ResponseWriter, r *http.Request) { return } - isPkey := searchPubkeyExactRE.MatchString(filter) - filterLen := len(filter) - - if filterLen != 0 && !isPkey { - errMsg := fmt.Errorf("error invalid pubkey characters or length") + if !isValidSubscriptionFilter(eventName, filter) { + errMsg := fmt.Errorf("error invalid filter, not pubkey or client") errFields := map[string]interface{}{ "filter": filter, "filter_len": len(filter)} @@ -2127,6 +2099,7 @@ func UserNotificationsUnsubscribe(w http.ResponseWriter, r *http.Request) { return } + filterLen := len(filter) if filterLen == 0 && !types.IsUserIndexed(eventName) { // no filter = add all my watched validators filter := db.WatchlistFilter{ @@ -2175,6 +2148,27 @@ func UserNotificationsUnsubscribe(w http.ResponseWriter, r *http.Request) { OKResponse(w, r) } +func isValidSubscriptionFilter(eventName types.EventName, filter string) bool { + ethClients := []string{"geth", "nethermind", "besu", "erigon", "teku", "prysm", "nimbus", "lighthouse", "lodestar", "rocketpool", "mev-boost"} + + isPkey := searchPubkeyExactRE.MatchString(filter) + + isClientName := false + for _, str := range ethClients { + if str == filter { + isClientName = true + break + } + } + + isClient := false + if eventName == types.EthClientUpdateEventName && isClientName { + isClient = true + } + + return len(filter) == 0 || isPkey || isClient +} + func UserNotificationsUnsubscribeByHash(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", "text/html") q := r.URL.Query() @@ -2184,8 +2178,8 @@ func UserNotificationsUnsubscribeByHash(w http.ResponseWriter, r *http.Request) hashes, ok := q["hash"] if !ok { - logger.Errorf("no query params given") - http.Error(w, "invalid request", 400) + logger.Warn("error no query params given") + http.Error(w, "Error: Missing parameter hash.", http.StatusBadRequest) return } @@ -2193,7 +2187,7 @@ func UserNotificationsUnsubscribeByHash(w http.ResponseWriter, r *http.Request) if err != nil { // return fmt.Errorf("error beginning transaction") logger.WithError(err).Errorf("error committing transacton") - http.Error(w, "error processing request", 500) + http.Error(w, "error processing request", http.StatusInternalServerError) return } defer tx.Rollback() @@ -2202,8 +2196,8 @@ func UserNotificationsUnsubscribeByHash(w http.ResponseWriter, r *http.Request) for _, hash := range hashes { hash = strings.Replace(hash, "0x", "", -1) if !utils.HashLikeRegex.MatchString(hash) { - logger.Errorf("error validating unsubscribe digest hashes") - http.Error(w, "error processing request", 500) + logger.Warn("error validating unsubscribe digest hashes") + http.Error(w, "Error: Invalid parameter hash entry.", http.StatusBadRequest) } b, _ := hex.DecodeString(hash) bHashes = append(bHashes, b) @@ -2212,14 +2206,14 @@ func UserNotificationsUnsubscribeByHash(w http.ResponseWriter, r *http.Request) _, err = tx.ExecContext(ctx, `DELETE from users_subscriptions where unsubscribe_hash = ANY($1)`, pq.ByteaArray(bHashes)) if err != nil { logger.Errorf("error deleting from users_subscriptions %v", err) - http.Error(w, "error processing request", 500) + http.Error(w, "error processing request", http.StatusInternalServerError) return } err = tx.Commit() if err != nil { logger.WithError(err).Errorf("error committing transacton") - http.Error(w, "error processing request", 500) + http.Error(w, "error processing request", http.StatusInternalServerError) return } @@ -3014,7 +3008,7 @@ func UserGlobalNotification(w http.ResponseWriter, r *http.Request) { } } -// LoginPost handles authenticating the user. +// UserGlobalNotificationPost handles the global notifications func UserGlobalNotificationPost(w http.ResponseWriter, r *http.Request) { isAdmin, _ := handleAdminPermissions(w, r) if !isAdmin { diff --git a/handlers/validator.go b/handlers/validator.go index 2e869f48a1..5d877818dc 100644 --- a/handlers/validator.go +++ b/handlers/validator.go @@ -13,6 +13,7 @@ import ( "eth2-exporter/utils" "fmt" "html/template" + "math" "math/big" "net/http" "sort" @@ -24,7 +25,6 @@ import ( "github.com/ethereum/go-ethereum/crypto" "github.com/lib/pq" protomath "github.com/protolambda/zrnt/eth2/util/math" - "golang.org/x/exp/slices" "golang.org/x/sync/errgroup" "github.com/gorilla/csrf" @@ -76,7 +76,15 @@ func Validator(w http.ResponseWriter, r *http.Request) { var err error latestEpoch := services.LatestEpoch() + latestProposedSlot := services.LatestProposedSlot() lastFinalizedEpoch := services.LatestFinalizedEpoch() + isPreGenesis := false + if latestEpoch == 0 { + latestEpoch = 1 + latestProposedSlot = 1 + lastFinalizedEpoch = 1 + isPreGenesis = true + } validatorPageData := types.ValidatorPageData{} @@ -242,7 +250,7 @@ func Validator(w http.ResponseWriter, r *http.Request) { } else { // Request came with a validator index number index, err = strconv.ParseUint(vars["index"], 10, 64) - if err != nil { + if err != nil || index > math.MaxInt32 { // index in postgres is limited to int validatorNotFound(data, w, r, vars, "") return } @@ -298,7 +306,7 @@ func Validator(w http.ResponseWriter, r *http.Request) { } validatorPageData.LastAttestationSlot = lastAttestationSlots[index] - lastStatsDay := services.LatestExportedStatisticDay() + lastStatsDay, lastStatsDayErr := services.LatestExportedStatisticDay() timings.BasicInfo = time.Since(timings.Start) @@ -342,18 +350,6 @@ func Validator(w http.ResponseWriter, r *http.Request) { validatorPageData.ExitTs = utils.EpochToTime(validatorPageData.ExitEpoch) validatorPageData.WithdrawableTs = utils.EpochToTime(validatorPageData.WithdrawableEpoch) - // Every validator is scheduled to issue an attestation once per epoch - // Hence we can calculate the number of attestations using the current epoch and the activation epoch - // Special care needs to be take for exited and pending validators - validatorPageData.AttestationsCount = validatorPageData.Epoch - validatorPageData.ActivationEpoch + 1 - if validatorPageData.ActivationEpoch > validatorPageData.Epoch { - validatorPageData.AttestationsCount = 0 - } - - if validatorPageData.ExitEpoch != 9223372036854775807 && validatorPageData.ExitEpoch <= validatorPageData.Epoch { - validatorPageData.AttestationsCount = validatorPageData.ExitEpoch - validatorPageData.ActivationEpoch - } - avgSyncInterval := uint64(getAvgSyncCommitteeInterval(1)) avgSyncIntervalAsDuration := time.Duration( utils.Config.Chain.ClConfig.SecondsPerSlot* @@ -365,7 +361,6 @@ func Validator(w http.ResponseWriter, r *http.Request) { if lastStatsDay > 30 { lowerBoundDay = lastStatsDay - 30 } - g := errgroup.Group{} g.Go(func() error { start := time.Now() @@ -373,11 +368,16 @@ func Validator(w http.ResponseWriter, r *http.Request) { timings.Charts = time.Since(start) }() - validatorPageData.IncomeHistoryChartData, err = db.GetValidatorIncomeHistoryChart([]uint64{index}, currency, lastFinalizedEpoch, lowerBoundDay) - + incomeHistoryChartData, err := db.GetValidatorIncomeHistoryChart([]uint64{index}, currency, lastFinalizedEpoch, lowerBoundDay) if err != nil { - return fmt.Errorf("error calling db.GetValidatorIncomeHistoryChart: %v", err) + return fmt.Errorf("error calling db.GetValidatorIncomeHistoryChart: %w", err) + } + + if isPreGenesis { + incomeHistoryChartData = make([]*types.ChartDataPoint, 0) } + + validatorPageData.IncomeHistoryChartData = incomeHistoryChartData return nil }) @@ -386,11 +386,13 @@ func Validator(w http.ResponseWriter, r *http.Request) { defer func() { timings.Charts = time.Since(start) }() - validatorPageData.ExecutionIncomeHistoryData, err = getExecutionChartData([]uint64{index}, currency, lowerBoundDay) + executionIncomeHistoryData, err := getExecutionChartData([]uint64{index}, currency, lowerBoundDay) if err != nil { - return fmt.Errorf("error calling getExecutionChartData: %v", err) + return fmt.Errorf("error calling getExecutionChartData: %w", err) } + + validatorPageData.ExecutionIncomeHistoryData = executionIncomeHistoryData return nil }) @@ -402,7 +404,7 @@ func Validator(w http.ResponseWriter, r *http.Request) { }() earnings, balances, err := GetValidatorEarnings([]uint64{index}, GetCurrency(r)) if err != nil { - return fmt.Errorf("error retrieving validator earnings: %v", err) + return fmt.Errorf("error retrieving validator earnings: %w", err) } // each income and apr variable is a struct of 3 fields: cl, el and total validatorPageData.Income1d = earnings.Income1d @@ -420,10 +422,6 @@ func Validator(w http.ResponseWriter, r *http.Request) { futureProposalEpoch = earnings.ProposalData.LastScheduledSlot / data.ChainConfig.SlotsPerEpoch } - if utils.Config.Frontend.Validator.ShowProposerRewards { - validatorPageData.IncomeProposerFormatted = &earnings.ProposerTotalFormatted - } - vbalance, ok := balances[validatorPageData.ValidatorIndex] if ok { validatorPageData.CurrentBalance = vbalance.Balance @@ -441,13 +439,13 @@ func Validator(w http.ResponseWriter, r *http.Request) { // get validator withdrawals withdrawalsCount, lastWithdrawalsEpoch, err := db.GetValidatorWithdrawalsCount(validatorPageData.Index) if err != nil { - return fmt.Errorf("error getting validator withdrawals count from db: %v", err) + return fmt.Errorf("error getting validator withdrawals count from db: %w", err) } validatorPageData.WithdrawalCount = withdrawalsCount blsChange, err := db.GetValidatorBLSChange(validatorPageData.Index) if err != nil { - return fmt.Errorf("error getting validator bls change from db: %v", err) + return fmt.Errorf("error getting validator bls change from db: %w", err) } validatorPageData.BLSChange = blsChange @@ -462,7 +460,7 @@ func Validator(w http.ResponseWriter, r *http.Request) { if stats != nil && stats.LatestValidatorWithdrawalIndex != nil && stats.TotalValidatorCount != nil && validatorPageData.IsWithdrawableAddress && (isFullWithdrawal || isPartialWithdrawal) { distance, err := GetWithdrawableCountFromCursor(validatorPageData.Epoch, validatorPageData.Index, *stats.LatestValidatorWithdrawalIndex) if err != nil { - return fmt.Errorf("error getting withdrawable validator count from cursor: %v", err) + return fmt.Errorf("error getting withdrawable validator count from cursor: %w", err) } timeToWithdrawal := utils.GetTimeToNextWithdrawal(distance) @@ -520,7 +518,7 @@ func Validator(w http.ResponseWriter, r *http.Request) { watchlist, err := db.GetTaggedValidators(filter) if err != nil { - return fmt.Errorf("error getting tagged validators from db: %v", err) + return fmt.Errorf("error getting tagged validators from db: %w", err) } validatorPageData.Watchlist = watchlist @@ -534,7 +532,7 @@ func Validator(w http.ResponseWriter, r *http.Request) { }() deposits, err := db.GetValidatorDeposits(validatorPageData.PublicKey) if err != nil { - return fmt.Errorf("error getting validator-deposits from db: %v", err) + return fmt.Errorf("error getting validator-deposits from db: %w", err) } validatorPageData.Deposits = deposits validatorPageData.DepositsCount = uint64(len(deposits.Eth1Deposits)) @@ -556,7 +554,7 @@ func Validator(w http.ResponseWriter, r *http.Request) { if validatorPageData.ActivationEpoch > 100_000_000 && validatorPageData.ActivationEligibilityEpoch < 100_000_000 { queueAhead, err := db.GetQueueAheadOfValidator(validatorPageData.Index) if err != nil { - return fmt.Errorf("failed to retrieve queue ahead of validator %v: %v", validatorPageData.ValidatorIndex, err) + return fmt.Errorf("failed to retrieve queue ahead of validator %v: %w", validatorPageData.ValidatorIndex, err) } validatorPageData.QueuePosition = queueAhead + 1 epochsToWait := queueAhead / *churnRate @@ -572,15 +570,38 @@ func Validator(w http.ResponseWriter, r *http.Request) { }) g.Go(func() error { + // Every validator is scheduled to issue an attestation once per epoch + // Hence we can calculate the number of attestations using the current epoch and the activation epoch + // Special care needs to be take for exited and pending validators + if validatorPageData.ExitEpoch != 9223372036854775807 && validatorPageData.ExitEpoch <= validatorPageData.Epoch { + validatorPageData.AttestationsCount = validatorPageData.ExitEpoch - validatorPageData.ActivationEpoch + } else if validatorPageData.ActivationEpoch > validatorPageData.Epoch || isPreGenesis { + validatorPageData.AttestationsCount = 0 + } else { + validatorPageData.AttestationsCount = validatorPageData.Epoch - validatorPageData.ActivationEpoch + 1 + + // Check if the latest epoch still needs to be attested (scheduled) and if so do not count it + attestationData, err := db.BigtableClient.GetValidatorAttestationHistory([]uint64{index}, validatorPageData.Epoch, validatorPageData.Epoch) + if err != nil { + return fmt.Errorf("error retrieving validator attestations data for epoch [%v] and validator index [%v]: %w", validatorPageData.Epoch, index, err) + } + + if len(attestationData[index]) > 0 && attestationData[index][0].Status == 0 { + validatorPageData.AttestationsCount-- + } + } + if validatorPageData.AttestationsCount > 0 { // get attestationStats from validator_stats attestationStats := struct { MissedAttestations uint64 `db:"missed_attestations"` }{} if lastStatsDay > 0 { - err = db.ReaderDb.Get(&attestationStats, "select coalesce(sum(missed_attestations), 0) as missed_attestations from validator_stats where validatorindex = $1", index) - if err != nil { - return fmt.Errorf("error retrieving validator attestationStats: %v", err) + err := db.ReaderDb.Get(&attestationStats, "SELECT missed_attestations_total AS missed_attestations FROM validator_stats WHERE validatorindex = $1 AND day = $2", index, lastStatsDay) + if err == sql.ErrNoRows { + logger.Warningf("no entry in validator_stats for validator index %v while lastStatsDay = %v", index, lastStatsDay) + } else if err != nil { + return fmt.Errorf("error retrieving validator attestationStats for index %v while lastStatsDay = %v: %w", index, lastStatsDay, err) } } @@ -590,19 +611,26 @@ func Validator(w http.ResponseWriter, r *http.Request) { // logger.Infof("retrieving attestations not yet in stats, lookback is %v", lookback) missedAttestations, err := db.BigtableClient.GetValidatorMissedAttestationHistory([]uint64{index}, lastFinalizedEpoch-uint64(lookback), lastFinalizedEpoch) if err != nil { - return fmt.Errorf("error retrieving validator attestations not in stats from bigtable: %v", err) + return fmt.Errorf("error retrieving validator attestations not in stats from bigtable: %w", err) } attestationStats.MissedAttestations += uint64(len(missedAttestations[index])) } - validatorPageData.MissedAttestationsCount = attestationStats.MissedAttestations - validatorPageData.ExecutedAttestationsCount = validatorPageData.AttestationsCount - validatorPageData.MissedAttestationsCount - validatorPageData.UnmissedAttestationsPercentage = float64(validatorPageData.ExecutedAttestationsCount) / float64(validatorPageData.AttestationsCount) + if isPreGenesis { + validatorPageData.MissedAttestationsCount = 0 + validatorPageData.ExecutedAttestationsCount = 0 + validatorPageData.UnmissedAttestationsPercentage = 1 + } else { + validatorPageData.MissedAttestationsCount = attestationStats.MissedAttestations + validatorPageData.ExecutedAttestationsCount = validatorPageData.AttestationsCount - validatorPageData.MissedAttestationsCount + validatorPageData.UnmissedAttestationsPercentage = float64(validatorPageData.ExecutedAttestationsCount) / float64(validatorPageData.AttestationsCount) + } } return nil }) g.Go(func() error { + var err error if validatorPageData.Slashed { var slashingInfo struct { Slot uint64 @@ -620,7 +648,7 @@ func Validator(w http.ResponseWriter, r *http.Request) { limit 1`, index) if err != nil { - return fmt.Errorf("error retrieving validator slashing info: %v", err) + return fmt.Errorf("error retrieving validator slashing info: %w", err) } validatorPageData.SlashedBy = slashingInfo.Slasher validatorPageData.SlashedAt = slashingInfo.Slot @@ -629,7 +657,7 @@ func Validator(w http.ResponseWriter, r *http.Request) { err = db.ReaderDb.Get(&validatorPageData.SlashingsCount, `select COALESCE(sum(attesterslashingscount) + sum(proposerslashingscount), 0) from blocks where blocks.proposer = $1 and blocks.status = '1'`, index) if err != nil { - return fmt.Errorf("error retrieving slashings-count: %v", err) + return fmt.Errorf("error retrieving slashings-count: %w", err) } return nil }) @@ -637,7 +665,7 @@ func Validator(w http.ResponseWriter, r *http.Request) { g.Go(func() error { eff, err := db.BigtableClient.GetValidatorEffectiveness([]uint64{index}, validatorPageData.Epoch-1) if err != nil { - return fmt.Errorf("error retrieving validator effectiveness: %v", err) + return fmt.Errorf("error retrieving validator effectiveness: %w", err) } if len(eff) > 1 { return fmt.Errorf("error retrieving validator effectiveness: invalid length %v", len(eff)) @@ -661,13 +689,13 @@ func Validator(w http.ResponseWriter, r *http.Request) { } allSyncPeriods := actualSyncPeriods - err = db.ReaderDb.Select(&allSyncPeriods, ` + err := db.ReaderDb.Select(&allSyncPeriods, ` SELECT period as period, (period*$1) as firstepoch, ((period+1)*$1)-1 as lastepoch FROM sync_committees WHERE validatorindex = $2 ORDER BY period desc`, utils.Config.Chain.ClConfig.EpochsPerSyncCommitteePeriod, index) if err != nil { - return fmt.Errorf("error getting sync participation count data of sync-assignments: %v", err) + return fmt.Errorf("error getting sync participation count data of sync-assignments: %w", err) } if len(allSyncPeriods) > 0 && allSyncPeriods[0].LastEpoch > latestEpoch { @@ -694,17 +722,21 @@ func Validator(w http.ResponseWriter, r *http.Request) { FROM validator_stats WHERE validatorindex = $1`, index) if err != nil { - return fmt.Errorf("error retrieving validator syncStats: %v", err) + return fmt.Errorf("error retrieving validator syncStats: %w", err) } } // if sync duties of last period haven't fully been exported yet, fetch remaining duties from bigtable lastExportedEpoch := (lastStatsDay+1)*utils.EpochsPerDay() - 1 + + if lastStatsDayErr == db.ErrNoStats { + lastExportedEpoch = 0 + } lastSyncPeriod := actualSyncPeriods[0] if lastSyncPeriod.LastEpoch > lastExportedEpoch { - res, err := db.BigtableClient.GetValidatorSyncDutiesHistory([]uint64{index}, lastExportedEpoch+1, latestEpoch) + res, err := db.BigtableClient.GetValidatorSyncDutiesHistory([]uint64{index}, (lastExportedEpoch+1)*utils.Config.Chain.ClConfig.SlotsPerEpoch, latestProposedSlot) if err != nil { - return fmt.Errorf("error retrieving validator sync participations data from bigtable: %v", err) + return fmt.Errorf("error retrieving validator sync participations data from bigtable: %w", err) } syncStatsBt := utils.AddSyncStats([]uint64{index}, res, nil) // if last sync period is the current one, add remaining scheduled slots @@ -731,7 +763,7 @@ func Validator(w http.ResponseWriter, r *http.Request) { maxPeriod := allSyncPeriods[0].Period expectedSyncCount, err := getExpectedSyncCommitteeSlots([]uint64{index}, latestEpoch) if err != nil { - return fmt.Errorf("error retrieving expected sync committee slots: %v", err) + return fmt.Errorf("error retrieving expected sync committee slots: %w", err) } if expectedSyncCount != 0 { validatorPageData.SyncLuck = float64(validatorPageData.ParticipatedSyncCountSlots+validatorPageData.MissedSyncCountSlots) / float64(expectedSyncCount) @@ -745,7 +777,7 @@ func Validator(w http.ResponseWriter, r *http.Request) { g.Go(func() error { // add rocketpool-data if available validatorPageData.Rocketpool = &types.RocketpoolValidatorPageData{} - err = db.ReaderDb.Get(validatorPageData.Rocketpool, ` + err := db.ReaderDb.Get(validatorPageData.Rocketpool, ` SELECT rplm.node_address AS node_address, rplm.address AS minipool_address, @@ -782,14 +814,14 @@ func Validator(w http.ResponseWriter, r *http.Request) { validatorPageData.Rocketpool.RocketscanUrl = "prater.rocketscan.io" } } else if err != nil && err != sql.ErrNoRows { - return fmt.Errorf("error getting rocketpool-data for validator for %v route: %v", r.URL.String(), err) + return fmt.Errorf("error getting rocketpool-data for validator for %v route: %w", r.URL.String(), err) } return nil }) err = g.Wait() if err != nil { - logger.Error(err) + utils.LogError(err, "error retrieving validator data", 0) http.Error(w, "Internal server error", http.StatusInternalServerError) return } @@ -851,8 +883,8 @@ func ValidatorDeposits(w http.ResponseWriter, r *http.Request) { pubkey, err := hex.DecodeString(strings.Replace(vars["pubkey"], "0x", "", -1)) if err != nil { - logger.Errorf("error parsing validator public key %v: %v", vars["pubkey"], err) - http.Error(w, "Internal server error", http.StatusInternalServerError) + logger.Warnf("error parsing validator public key %v: %v", vars["pubkey"], err) + http.Error(w, "Error: Invalid parameter public key.", http.StatusBadRequest) return } @@ -877,13 +909,17 @@ func ValidatorAttestationInclusionEffectiveness(w http.ResponseWriter, r *http.R vars := mux.Vars(r) index, err := strconv.ParseUint(vars["index"], 10, 64) - if err != nil { - logger.Errorf("error parsing validator index: %v", err) - http.Error(w, "Internal server error", http.StatusInternalServerError) + if err != nil || index > math.MaxInt32 { // index in postgres is limited to int + logger.Warnf("error parsing validator index: %v", err) + http.Error(w, "Error: Invalid parameter validator index.", http.StatusBadRequest) return } + epoch := services.LatestEpoch() + if epoch > 0 { + epoch = epoch - 1 + } - eff, err := db.BigtableClient.GetValidatorEffectiveness([]uint64{index}, services.LatestEpoch()-1) + eff, err := db.BigtableClient.GetValidatorEffectiveness([]uint64{index}, epoch) if err != nil { logger.Errorf("error retrieving validator effectiveness: %v", err) http.Error(w, "Internal server error", http.StatusInternalServerError) @@ -922,9 +958,9 @@ func ValidatorProposedBlocks(w http.ResponseWriter, r *http.Request) { vars := mux.Vars(r) index, err := strconv.ParseUint(vars["index"], 10, 64) - if err != nil { - logger.Errorf("error parsing validator index: %v", err) - http.Error(w, "Internal server error", http.StatusInternalServerError) + if err != nil || index > math.MaxInt32 { // index in postgres is limited to int + logger.Warnf("error parsing validator index: %v", err) + http.Error(w, "Error: Invalid parameter validator index.", http.StatusBadRequest) return } @@ -932,20 +968,20 @@ func ValidatorProposedBlocks(w http.ResponseWriter, r *http.Request) { draw, err := strconv.ParseUint(q.Get("draw"), 10, 64) if err != nil { - logger.Errorf("error converting datatables data parameter from string to int: %v", err) - http.Error(w, "Internal server error", http.StatusInternalServerError) + logger.Warnf("error converting datatables draw parameter from string to int: %v", err) + http.Error(w, "Error: Missing or invalid parameter draw", http.StatusBadRequest) return } start, err := strconv.ParseUint(q.Get("start"), 10, 64) if err != nil { - logger.Errorf("error converting datatables start parameter from string to int: %v", err) - http.Error(w, "Internal server error", http.StatusInternalServerError) + logger.Warnf("error converting datatables start parameter from string to int: %v", err) + http.Error(w, "Error: Missing or invalid parameter start", http.StatusBadRequest) return } length, err := strconv.ParseUint(q.Get("length"), 10, 64) if err != nil { - logger.Errorf("error converting datatables length parameter from string to int: %v", err) - http.Error(w, "Internal server error", http.StatusInternalServerError) + logger.Warnf("error converting datatables length parameter from string to int: %v", err) + http.Error(w, "Error: Missing or invalid parameter length", http.StatusBadRequest) return } if length > 100 { @@ -1043,9 +1079,9 @@ func ValidatorAttestations(w http.ResponseWriter, r *http.Request) { vars := mux.Vars(r) index, err := strconv.ParseUint(vars["index"], 10, 64) - if err != nil { - logger.Errorf("error parsing validator index: %v", err) - http.Error(w, "Internal server error", http.StatusInternalServerError) + if err != nil || index > math.MaxInt32 { // index in postgres is limited to int + logger.Warnf("error parsing validator index: %v", err) + http.Error(w, "Error: Invalid parameter validator index.", http.StatusBadRequest) return } @@ -1053,14 +1089,14 @@ func ValidatorAttestations(w http.ResponseWriter, r *http.Request) { draw, err := strconv.ParseUint(q.Get("draw"), 10, 64) if err != nil { - logger.Errorf("error converting datatables data parameter from string to int: %v", err) - http.Error(w, "Internal server error", http.StatusInternalServerError) + logger.Warnf("error converting datatables draw parameter from string to int: %v", err) + http.Error(w, "Error: Missing or invalid parameter draw", http.StatusBadRequest) return } start, err := strconv.ParseInt(q.Get("start"), 10, 64) if err != nil { - logger.Errorf("error converting datatables start parameter from string to int: %v", err) - http.Error(w, "Internal server error", http.StatusInternalServerError) + logger.Warnf("error converting datatables start parameter from string to int: %v", err) + http.Error(w, "Error: Missing or invalid parameter start", http.StatusBadRequest) return } @@ -1093,10 +1129,19 @@ func ValidatorAttestations(w http.ResponseWriter, r *http.Request) { tableData := [][]interface{}{} if totalCount > 0 { - endEpoch := uint64(int64(lastAttestationEpoch) - start) - attestationData, err := db.BigtableClient.GetValidatorAttestationHistory([]uint64{index}, endEpoch-uint64(length)+1, endEpoch) + endEpoch := int64(lastAttestationEpoch) - start + if endEpoch < 0 { + endEpoch = 0 + } + + startEpoch := endEpoch - int64(length) + 1 + if startEpoch < 0 { + startEpoch = 0 + } + + attestationData, err := db.BigtableClient.GetValidatorAttestationHistory([]uint64{index}, uint64(startEpoch), uint64(endEpoch)) if err != nil { - logger.Errorf("error retrieving validator attestations data: %v", err) + logger.Errorf("error retrieving bigtable validator attestations data: %v", err) http.Error(w, "Internal server error", http.StatusInternalServerError) return } @@ -1140,9 +1185,9 @@ func ValidatorWithdrawals(w http.ResponseWriter, r *http.Request) { vars := mux.Vars(r) index, err := strconv.ParseUint(vars["index"], 10, 64) - if err != nil { - logger.Errorf("error parsing validator index: %v", err) - http.Error(w, "Internal server error", http.StatusInternalServerError) + if err != nil || index > math.MaxInt32 { // index in postgres is limited to int + logger.Warnf("error parsing validator index: %v", err) + http.Error(w, "Error: Invalid parameter validator index.", http.StatusBadRequest) return } @@ -1150,14 +1195,14 @@ func ValidatorWithdrawals(w http.ResponseWriter, r *http.Request) { draw, err := strconv.ParseUint(q.Get("draw"), 10, 64) if err != nil { - logger.Errorf("error converting datatables data parameter from string to int: %v", err) - http.Error(w, "Internal server error", http.StatusInternalServerError) + logger.Warnf("error converting datatables draw parameter from string to int: %v", err) + http.Error(w, "Error: Missing or invalid parameter draw", http.StatusBadRequest) return } start, err := strconv.ParseUint(q.Get("start"), 10, 64) if err != nil { - logger.Errorf("error converting datatables start parameter from string to int: %v", err) - http.Error(w, "Internal server error", http.StatusInternalServerError) + logger.Warnf("error converting datatables start parameter from string to int: %v", err) + http.Error(w, "Error: Missing or invalid parameter start", http.StatusBadRequest) return } @@ -1227,9 +1272,9 @@ func ValidatorSlashings(w http.ResponseWriter, r *http.Request) { vars := mux.Vars(r) index, err := strconv.ParseUint(vars["index"], 10, 64) - if err != nil { - logger.Errorf("error parsing validator index: %v", err) - http.Error(w, "Internal server error", http.StatusInternalServerError) + if err != nil || index > math.MaxInt32 { // index in postgres is limited to int + logger.Warnf("error parsing validator index: %v", err) + http.Error(w, "Error: Invalid parameter validator index.", http.StatusBadRequest) return } @@ -1237,8 +1282,8 @@ func ValidatorSlashings(w http.ResponseWriter, r *http.Request) { draw, err := strconv.ParseUint(q.Get("draw"), 10, 64) if err != nil { - logger.Errorf("error converting datatables data parameter from string to int: %v", err) - http.Error(w, "Internal server error", http.StatusInternalServerError) + logger.Warnf("error converting datatables draw parameter from string to int: %v", err) + http.Error(w, "Error: Missing or invalid parameter draw", http.StatusBadRequest) return } @@ -1379,14 +1424,16 @@ func sanitizeMessage(msg string) ([]byte, error) { } } -func ValidatorSave(w http.ResponseWriter, r *http.Request) { - pubkey := r.FormValue("pubkey") +func SaveValidatorName(w http.ResponseWriter, r *http.Request) { + vars := mux.Vars(r) + + pubkey := vars["pubkey"] pubkey = strings.ToLower(pubkey) pubkey = strings.Replace(pubkey, "0x", "", -1) pubkeyDecoded, err := hex.DecodeString(pubkey) if err != nil { - logger.Errorf("error parsing submitted pubkey %v: %v", pubkey, err) + logger.Warnf("error parsing submitted pubkey %v: %v", pubkey, err) utils.SetFlash(w, r, validatorEditFlash, "Error: the provided signature is invalid") http.Redirect(w, r, "/validator/"+pubkey, http.StatusMovedPermanently) return @@ -1487,7 +1534,6 @@ func ValidatorSave(w http.ResponseWriter, r *http.Request) { utils.SetFlash(w, r, validatorEditFlash, "Error: the provided signature is invalid") http.Redirect(w, r, "/validator/"+pubkey, http.StatusMovedPermanently) } - } // ValidatorHistory returns a validators history in json @@ -1499,9 +1545,9 @@ func ValidatorHistory(w http.ResponseWriter, r *http.Request) { vars := mux.Vars(r) index, err := strconv.ParseUint(vars["index"], 10, 64) - if err != nil { - logger.Errorf("error parsing validator index: %v", err) - http.Error(w, "Internal server error", http.StatusInternalServerError) + if err != nil || index > math.MaxInt32 { // index in postgres is limited to int + logger.Warnf("error parsing validator index: %v", err) + http.Error(w, "Error: Invalid parameter validator index.", http.StatusBadRequest) return } @@ -1509,15 +1555,15 @@ func ValidatorHistory(w http.ResponseWriter, r *http.Request) { draw, err := strconv.ParseUint(q.Get("draw"), 10, 64) if err != nil { - logger.Errorf("error converting datatables data parameter from string to int: %v", err) - http.Error(w, "Internal server error", http.StatusInternalServerError) + logger.Warnf("error converting datatables draw parameter from string to int: %v", err) + http.Error(w, "Error: Missing or invalid parameter draw", http.StatusBadRequest) return } start, err := strconv.ParseUint(q.Get("start"), 10, 64) if err != nil { - logger.Errorf("error converting datatables start parameter from string to int: %v", err) - http.Error(w, "Internal server error", http.StatusInternalServerError) + logger.Warnf("error converting datatables start parameter from string to int: %v", err) + http.Error(w, "Error: Missing or invalid parameter start", http.StatusBadRequest) return } @@ -1547,7 +1593,11 @@ func ValidatorHistory(w http.ResponseWriter, r *http.Request) { start = uint64((maxPages - 1) * pageLength) } - currentEpoch := services.LatestEpoch() - 1 + currentEpoch := services.LatestEpoch() + + if currentEpoch != 0 { + currentEpoch = currentEpoch - 1 + } var postExitEpochs uint64 = 0 // for an exited validator we show the history until his exit or (in rare cases) until his last sync / propose duties are finished if activationAndExitEpoch.ExitEpoch != 9223372036854775807 && currentEpoch > (activationAndExitEpoch.ExitEpoch-1) { @@ -1580,7 +1630,6 @@ func ValidatorHistory(w http.ResponseWriter, r *http.Request) { } tableData := make([][]interface{}, 0) - if postExitEpochs > 0 { startEpoch := currentEpoch + 1 endEpoch := startEpoch + postExitEpochs @@ -1591,6 +1640,10 @@ func ValidatorHistory(w http.ResponseWriter, r *http.Request) { // if there are additional epochs with duties we have to go through all of them as there can be gaps (after the exit before the duty) for i := endEpoch; i >= startEpoch; i-- { + if i > endEpoch { + break + } + if incomeDetails[index] == nil || incomeDetails[index][i] == nil { continue } @@ -1621,6 +1674,11 @@ func ValidatorHistory(w http.ResponseWriter, r *http.Request) { } for epoch := endEpoch; epoch >= startEpoch && len(tableData) < pageLength; epoch-- { + + if epoch > endEpoch { + break + } + if incomeDetails[index] == nil || incomeDetails[index][epoch] == nil { if epoch <= endEpoch { rewardsStr := "pending..." @@ -1639,6 +1697,7 @@ func ValidatorHistory(w http.ResponseWriter, r *http.Request) { continue } tableData = append(tableData, icomeToTableData(epoch, incomeDetails[index][epoch], withdrawalMap[epoch], currency)) + } } @@ -1747,18 +1806,16 @@ func ValidatorStatsTable(w http.ResponseWriter, r *http.Request) { data := InitPageData(w, r, "validators", "/validators", "", templateFiles) // Request came with a hash - if strings.Contains(vars["index"], "0x") || len(vars["index"]) == 96 { - pubKey, err := hex.DecodeString(strings.Replace(vars["index"], "0x", "", -1)) + if utils.IsHash(vars["index"]) { + pubKey, err := hex.DecodeString(strings.TrimPrefix(vars["index"], "0x")) if err != nil { logger.Errorf("error parsing validator public key %v: %v", vars["index"], err) - validatorNotFound(data, w, r, vars, "/stats") - return } index, err = db.GetValidatorIndex(pubKey) if err != nil { - logger.Errorf("error parsing validator pubkey: %v", err) + logger.Warnf("error parsing validator pubkey: %v", err) validatorNotFound(data, w, r, vars, "/stats") return } @@ -1766,8 +1823,8 @@ func ValidatorStatsTable(w http.ResponseWriter, r *http.Request) { // Request came with a validator index number index, err = strconv.ParseUint(vars["index"], 10, 64) // Request is not a valid index number - if err != nil { - logger.Errorf("error parsing validator index: %v", err) + if err != nil || index > math.MaxInt32 { // index in postgres is limited to int + logger.Warnf("error parsing validator index: %v", err) validatorNotFound(data, w, r, vars, "/stats") return } @@ -1829,9 +1886,9 @@ func ValidatorSync(w http.ResponseWriter, r *http.Request) { vars := mux.Vars(r) validatorIndex, err := strconv.ParseUint(vars["index"], 10, 64) - if err != nil { - logger.Errorf("error parsing validator index: %v", err) - http.Error(w, "Internal server error", http.StatusInternalServerError) + if err != nil || validatorIndex > math.MaxInt32 { // index in postgres is limited to int + logger.Warnf("error parsing validator index: %v", err) + http.Error(w, "Error: Invalid parameter validator index.", http.StatusBadRequest) return } @@ -1839,235 +1896,102 @@ func ValidatorSync(w http.ResponseWriter, r *http.Request) { draw, err := strconv.ParseUint(q.Get("draw"), 10, 64) if err != nil { - logger.Errorf("error converting datatables data draw-parameter from string to int: %v", err) - http.Error(w, "Internal server error", http.StatusInternalServerError) + logger.Warnf("error converting datatables draw parameter from string to int: %v", err) + http.Error(w, "Error: Missing or invalid parameter draw", http.StatusBadRequest) return } start, err := strconv.ParseUint(q.Get("start"), 10, 64) if err != nil { - logger.Errorf("error converting datatables start start-parameter from string to int: %v", err) - http.Error(w, "Internal server error", http.StatusInternalServerError) + logger.Warnf("error converting datatables start parameter from string to int: %v", err) + http.Error(w, "Error: Missing or invalid parameter start", http.StatusBadRequest) return } length, err := strconv.ParseUint(q.Get("length"), 10, 64) if err != nil { - logger.Errorf("error converting datatables length length-parameter from string to int: %v", err) - http.Error(w, "Internal server error", http.StatusInternalServerError) + logger.Warnf("error converting datatables length parameter from string to int: %v", err) + http.Error(w, "Error: Missing or invalid parameter length", http.StatusBadRequest) return } if length > 100 { length = 100 } - ascOrdering := q.Get("order[0][dir]") == "asc" + // descOrdering := q.Get("order[0][dir]") == "desc" // retrieve all sync periods for this validator - // ordering is descending for now - var syncPeriods []struct { - Period uint64 `db:"period"` - StartEpoch uint64 `db:"startepoch"` - EndEpoch uint64 `db:"endepoch"` - } - tempSyncPeriods := syncPeriods + var syncPeriods []uint64 = []uint64{} - err = db.ReaderDb.Select(&tempSyncPeriods, ` - SELECT period as period, (period*$1) as endepoch, ((period+1)*$1)-1 as startepoch + err = db.ReaderDb.Select(&syncPeriods, ` + SELECT period FROM sync_committees - WHERE validatorindex = $2 - ORDER BY period desc`, utils.Config.Chain.ClConfig.EpochsPerSyncCommitteePeriod, validatorIndex) + WHERE validatorindex = $1 + ORDER BY period desc`, validatorIndex) if err != nil { logger.WithError(err).Errorf("error getting sync tab count data of sync-assignments") http.Error(w, "Internal server error", http.StatusInternalServerError) return } - latestEpoch := services.LatestEpoch() - - //remove scheduled committees - for i, syncPeriod := range tempSyncPeriods { - if syncPeriod.EndEpoch <= latestEpoch { - syncPeriods = tempSyncPeriods[i:] - break - } - } - - // set latest epoch of this validators latest sync period to current epoch if latest sync epoch has yet to happen - var diffToLatestEpoch uint64 = 0 - if latestEpoch < syncPeriods[0].StartEpoch { - diffToLatestEpoch = syncPeriods[0].StartEpoch - latestEpoch - syncPeriods[0].StartEpoch = latestEpoch - } - // total count of sync duties for this validator - totalCount := (uint64(len(syncPeriods))*utils.Config.Chain.ClConfig.EpochsPerSyncCommitteePeriod - diffToLatestEpoch) * utils.Config.Chain.ClConfig.SlotsPerEpoch + totalCount := uint64(0) tableData := [][]interface{}{} - if totalCount > 0 && start <= totalCount { - // if ordering is ascending, reverse sync period slice & swap start and end epoch of each period - if ascOrdering { - utils.ReverseSlice(syncPeriods) - for i := range syncPeriods { - syncPeriods[i].StartEpoch, syncPeriods[i].EndEpoch = syncPeriods[i].EndEpoch, syncPeriods[i].StartEpoch - } - } - // syncPeriods[0].startEpoch will always be the epoch shown on page 1, regardless of the ordering - // meaning that for descending ordering, syncPeriods[0].startEpoch will be the chronologically latest epoch of this validators lastest sync period - // and for ascending ordering, syncPeriods[0].startEpoch will be the chronologically earliest epoch of this validators first sync period - - // set functions for moving away from start and back to start (with start being page 1) - // depending on the ordering, this means either going up or down in epoch number - var moveAway func(uint64, uint64) uint64 - var moveBack func(uint64, uint64) uint64 - var IsFurtherAway func(uint64, uint64) bool - if ascOrdering { - moveAway = func(a uint64, b uint64) uint64 { - return a + b - } - moveBack = func(a uint64, b uint64) uint64 { - return a - b - } - IsFurtherAway = func(a uint64, b uint64) bool { - return a > b - } - } else { - moveAway = func(a uint64, b uint64) uint64 { - return a - b - } - moveBack = func(a uint64, b uint64) uint64 { - return a + b - } - IsFurtherAway = func(a uint64, b uint64) bool { - return a < b - } - } - diffValue := func(a uint64, b uint64) uint64 { - if a >= b { - return a - b - } else { - return b - a - } - } + if len(syncPeriods) > 0 { - // amount of epochs moved away from start epoch - epochOffset := (start / utils.Config.Chain.ClConfig.SlotsPerEpoch) - // amount of distinct consecutive epochs shown on this page - epochsDiff := ((start + length) / utils.Config.Chain.ClConfig.SlotsPerEpoch) - epochOffset + latestProposedSlot := services.LatestProposedSlot() - shownEpochIndex := 0 - // first epoch containing the duties shown on this page - firstShownEpoch := moveAway(syncPeriods[shownEpochIndex].StartEpoch, epochOffset) + slots := make([]uint64, 0, utils.Config.Chain.ClConfig.EpochsPerSyncCommitteePeriod*utils.Config.Chain.ClConfig.SlotsPerEpoch*uint64(len(syncPeriods))) - // handle first shown epoch being in the next sync period - for IsFurtherAway(firstShownEpoch, syncPeriods[shownEpochIndex].EndEpoch) { - overshoot := firstShownEpoch - syncPeriods[shownEpochIndex].EndEpoch - shownEpochIndex++ - firstShownEpoch = syncPeriods[shownEpochIndex].StartEpoch + moveBack(overshoot, 1) - } + for _, period := range syncPeriods { + firstEpoch := utils.FirstEpochOfSyncPeriod(period) + lastEpoch := firstEpoch + utils.Config.Chain.ClConfig.EpochsPerSyncCommitteePeriod - 1 - // last epoch containing the duties shown on this page - lastShownEpoch := moveAway(firstShownEpoch, epochsDiff) - // handle overflow on last page for validators that were part of the very first sync period starting during epoch 0 - if !ascOrdering && lastShownEpoch > firstShownEpoch { - lastShownEpoch = 0 - length = utils.Config.Chain.ClConfig.SlotsPerEpoch - (start % utils.Config.Chain.ClConfig.SlotsPerEpoch) - } - // amount of epochs fetched by bigtable - limit := diffValue(firstShownEpoch, lastShownEpoch) + 1 + firstSlot := firstEpoch * utils.Config.Chain.ClConfig.SlotsPerEpoch + lastSlot := (lastEpoch+1)*utils.Config.Chain.ClConfig.SlotsPerEpoch - 1 - var nextPeriodLimit uint64 = 0 - if IsFurtherAway(lastShownEpoch, syncPeriods[shownEpochIndex].EndEpoch) { - if IsFurtherAway(lastShownEpoch, syncPeriods[len(syncPeriods)-1].EndEpoch) { - // handle showing the last page, which may hold less than 'length' amount of rows - length = utils.Config.Chain.ClConfig.SlotsPerEpoch - (start % utils.Config.Chain.ClConfig.SlotsPerEpoch) - } else { - // handle crossing sync periods on the same page (i.e. including the earliest and latest slot of two sync periods from this validator) - overshoot := diffValue(lastShownEpoch, syncPeriods[shownEpochIndex].EndEpoch) - lastShownEpoch = syncPeriods[shownEpochIndex+1].StartEpoch + overshoot - 1 - limit -= overshoot - nextPeriodLimit = overshoot + // logger.Infof("processing sync period %v epoch (%v/%v) slots (%v/%v)", period, firstEpoch, lastEpoch, firstSlot, lastSlot) + for slot := lastSlot; slot >= firstSlot && (slot <= lastSlot /* guards against underflows */); slot-- { + if slot > latestProposedSlot || utils.EpochOfSlot(slot) < utils.Config.Chain.ClConfig.AltairForkEpoch { + continue + } + slots = append(slots, slot) } } - // retrieve sync duties from bigtable - var startEpoch, endEpoch uint64 - if ascOrdering { - startEpoch = firstShownEpoch - endEpoch = firstShownEpoch + limit - 1 - } else { - startEpoch = firstShownEpoch - (limit - 1) - endEpoch = firstShownEpoch - } - syncDuties, err := db.BigtableClient.GetValidatorSyncDutiesHistoryOrdered([]uint64{}, startEpoch, endEpoch, ascOrdering) - if err != nil { - logger.Errorf("error retrieving validator sync duty data from bigtable: %v", err) - http.Error(w, "Internal server error", http.StatusInternalServerError) - return - } - - syncDutiesValidator := syncDuties[validatorIndex] - syncDutiesAllValidators := make([]uint64, limit*utils.Config.Chain.ClConfig.SlotsPerEpoch) - for _, duties := range syncDuties { - for idx := range syncDutiesValidator { - slot := syncDutiesValidator[idx].Slot - index := slices.IndexFunc[[]*types.ValidatorSyncParticipation](duties, func(duty *types.ValidatorSyncParticipation) bool { - return duty.Slot == slot - }) + // if ordering is desc, reverse sync slots + totalCount = uint64(len(slots)) - if index >= 0 && duties[index].Status == 1 { - syncDutiesAllValidators[idx]++ - } - } + startIndex := start + length - 1 + if startIndex > uint64(len(slots)-1) { + startIndex = uint64(len(slots) - 1) } + endIndex := start - if nextPeriodLimit != 0 { - if ascOrdering { - startEpoch = lastShownEpoch - (nextPeriodLimit - 1) - endEpoch = lastShownEpoch - } else { - startEpoch = lastShownEpoch - endEpoch = lastShownEpoch + nextPeriodLimit - 1 - } - nextPeriodSyncDuties, err := db.BigtableClient.GetValidatorSyncDutiesHistoryOrdered([]uint64{}, startEpoch, endEpoch, ascOrdering) - if err != nil { - logger.Errorf("error retrieving second validator sync duty data from bigtable: %v", err) - http.Error(w, "Internal server error", http.StatusInternalServerError) - return - } + endSlot := slots[endIndex] + startSlot := slots[startIndex] - nextPeriodSyncDutiesValidator := nextPeriodSyncDuties[validatorIndex] - nextPeriodSyncDutiesAllValidators := make([]uint64, nextPeriodLimit*utils.Config.Chain.ClConfig.SlotsPerEpoch) - for _, duties := range nextPeriodSyncDuties { - for idx := range nextPeriodSyncDutiesValidator { - slot := nextPeriodSyncDutiesValidator[idx].Slot - index := slices.IndexFunc[[]*types.ValidatorSyncParticipation](duties, func(duty *types.ValidatorSyncParticipation) bool { - return duty.Slot == slot - }) + // logger.Infof("retrieving sync duty history for validator %v and slots %v-%v (%v-%v)", validatorIndex, startSlot, endSlot, startIndex, endIndex) + syncDuties, err := db.BigtableClient.GetValidatorSyncDutiesHistory([]uint64{validatorIndex}, startSlot, endSlot) - if index >= 0 && duties[index].Status == 1 { - nextPeriodSyncDutiesAllValidators[idx]++ - } - } - } - syncDutiesValidator = append(syncDutiesValidator, nextPeriodSyncDutiesValidator...) - syncDutiesAllValidators = append(syncDutiesAllValidators, nextPeriodSyncDutiesAllValidators...) + if err != nil { + utils.LogError(fmt.Errorf("error retrieving validator [%v] sync duty data from bigtable for slots [%v-%v]: %w", validatorIndex, startSlot, endSlot, err), "", 0) + http.Error(w, "Internal server error", http.StatusInternalServerError) + return } - var missedSyncSlots []uint64 - for _, synDuty := range syncDutiesValidator { - slotTime := utils.SlotToTime(synDuty.Slot) - - if synDuty.Status == 0 && time.Since(slotTime) <= time.Minute { - synDuty.Status = 2 // scheduled - } + // spew.Dump(syncDuties[validatorIndex]) + // Search for the missed slots (status = 2), to see if it was only our validator that missed the slot or if the block was missed + slotsRange := slots[endIndex : startIndex+1] - if synDuty.Status == 0 { - missedSyncSlots = append(missedSyncSlots, synDuty.Slot) - } + participations, err := db.BigtableClient.GetSyncParticipationBySlotRange(startSlot, endSlot) + if err != nil { + utils.LogError(fmt.Errorf("error retrieving validator [%v] sync participation data from bigtable for slots [%v-%v]: %w", validatorIndex, startSlot, endSlot, err), "", 0) + http.Error(w, "Internal server error", http.StatusInternalServerError) + return } - // Search for the missed slots (status = 2) missedSlots := []uint64{} - err = db.ReaderDb.Select(&missedSlots, `SELECT slot FROM blocks WHERE slot = ANY($1) AND status = '2'`, missedSyncSlots) + err = db.ReaderDb.Select(&missedSlots, `SELECT slot FROM blocks WHERE slot = ANY($1) AND status = '2'`, slotsRange) if err != nil { logger.WithError(err).Errorf("error getting missed slots data") http.Error(w, "Internal server error", http.StatusInternalServerError) @@ -2078,30 +2002,48 @@ func ValidatorSync(w http.ResponseWriter, r *http.Request) { missedSlotsMap[slot] = true } - // sanity check for right amount of slots in response - if uint64(len(syncDutiesValidator))%utils.Config.Chain.ClConfig.SlotsPerEpoch == 0 { - // extract correct slots - tableData = make([][]interface{}, length) - for dataIndex, slotIndex := 0, start%utils.Config.Chain.ClConfig.SlotsPerEpoch; slotIndex < protomath.MinU64((start%utils.Config.Chain.ClConfig.SlotsPerEpoch)+length, uint64(len(syncDuties))); dataIndex, slotIndex = dataIndex+1, slotIndex+1 { - participation := uint64(0) - if uint64(len(syncDutiesAllValidators)) > slotIndex { - participation = syncDutiesAllValidators[slotIndex] - } + // extract correct slots + tableData = make([][]interface{}, 0, length) + for slot := endSlot; slot >= startSlot; slot-- { - slot := syncDutiesValidator[slotIndex].Slot - epoch := utils.EpochOfSlot(slot) - status := syncDutiesValidator[slotIndex].Status - if _, ok := missedSlotsMap[slot]; ok { - status = 3 - } + epoch := utils.EpochOfSlot(slot) + participation := participations[slot] - tableData[dataIndex] = []interface{}{ - fmt.Sprintf("%d", utils.SyncPeriodOfEpoch(epoch)), - utils.FormatEpoch(epoch), - utils.FormatBlockSlot(slot), - utils.FormatSyncParticipationStatus(status, slot), - utils.FormatSyncParticipations(participation), - } + if syncDuties[validatorIndex] == nil { + tableData = append(tableData, + []interface{}{ + fmt.Sprintf("%d", utils.SyncPeriodOfEpoch(epoch)), + utils.FormatEpoch(epoch), + utils.FormatBlockSlot(slot), + utils.FormatSyncParticipationStatus(0, slot), + utils.FormatSyncParticipations(participation), + }, + ) + } + + // if utils.SlotToTime(slot).Before(time.Now()) { + // continue + // } + + status := uint64(0) + + if syncDuties[validatorIndex][slot] != nil { + status = syncDuties[validatorIndex][slot].Status + } + if _, ok := missedSlotsMap[slot]; ok { + status = 3 + } + + tableData = append(tableData, []interface{}{ + fmt.Sprintf("%d", utils.SyncPeriodOfEpoch(epoch)), + utils.FormatEpoch(epoch), + utils.FormatBlockSlot(slot), + utils.FormatSyncParticipationStatus(status, slot), + utils.FormatSyncParticipations(participation), + }) + + if slot == 0 { + break } } } diff --git a/handlers/validatorRewards.go b/handlers/validatorRewards.go index 657417fbc7..02ba44c8d1 100644 --- a/handlers/validatorRewards.go +++ b/handlers/validatorRewards.go @@ -116,20 +116,24 @@ func RewardsHistoricalData(w http.ResponseWriter, r *http.Request) { currency := q.Get("currency") - var start uint64 = 0 - var end uint64 = 0 + // Set the default start and end time to the first day + t := time.Unix(int64(utils.Config.Chain.GenesisTimestamp), 0) + startGenesisDay := uint64(time.Date(t.Year(), t.Month(), t.Day(), 0, 0, 0, 0, time.UTC).Unix()) + var start uint64 = startGenesisDay + var end uint64 = startGenesisDay + dateRange := strings.Split(q.Get("days"), "-") if len(dateRange) == 2 { - start, err = strconv.ParseUint(dateRange[0], 10, 64) - if err != nil { - logger.Errorf("error retrieving days range %v", err) - http.Error(w, "Invalid query", 400) + start, err = strconv.ParseUint(dateRange[0], 10, 32) //Limit to uint32 for postgres + if err != nil || start < startGenesisDay { + logger.Warnf("error parsing days range: %v", err) + http.Error(w, "Error: Invalid parameter days.", http.StatusBadRequest) return } - end, err = strconv.ParseUint(dateRange[1], 10, 64) - if err != nil { - logger.Errorf("error retrieving days range %v", err) - http.Error(w, "Invalid query", 400) + end, err = strconv.ParseUint(dateRange[1], 10, 32) //Limit to uint32 for postgres + if err != nil || end < startGenesisDay { + logger.Warnf("error parsing days range: %v", err) + http.Error(w, "Error: Invalid parameter days.", http.StatusBadRequest) return } } @@ -155,20 +159,24 @@ func DownloadRewardsHistoricalData(w http.ResponseWriter, r *http.Request) { currency := q.Get("currency") - var start uint64 = 0 - var end uint64 = 0 + // Set the default start and end time to the first day + t := time.Unix(int64(utils.Config.Chain.GenesisTimestamp), 0) + startGenesisDay := uint64(time.Date(t.Year(), t.Month(), t.Day(), 0, 0, 0, 0, time.UTC).Unix()) + var start uint64 = startGenesisDay + var end uint64 = startGenesisDay + dateRange := strings.Split(q.Get("days"), "-") if len(dateRange) == 2 { - start, err = strconv.ParseUint(dateRange[0], 10, 64) - if err != nil { - logger.Errorf("error retrieving days range %v", err) - http.Error(w, "Invalid query", 400) + start, err = strconv.ParseUint(dateRange[0], 10, 32) //Limit to uint32 for postgres + if err != nil || start < startGenesisDay { + logger.Warnf("error parsing days range: %v", err) + http.Error(w, "Error: Invalid parameter days.", http.StatusBadRequest) return } - end, err = strconv.ParseUint(dateRange[1], 10, 64) - if err != nil { - logger.Errorf("error retrieving days range %v", err) - http.Error(w, "Invalid query", 400) + end, err = strconv.ParseUint(dateRange[1], 10, 32) //Limit to uint32 for postgres + if err != nil || end < startGenesisDay { + logger.Warnf("error parsing days range: %v", err) + http.Error(w, "Error: Invalid parameter days.", http.StatusBadRequest) return } } diff --git a/handlers/validators.go b/handlers/validators.go index d6d161a82a..fcaa03868e 100644 --- a/handlers/validators.go +++ b/handlers/validators.go @@ -183,19 +183,23 @@ func parseValidatorsDataQueryParams(r *http.Request) (*ValidatorsDataQueryParams draw, err := strconv.ParseUint(q.Get("draw"), 10, 64) if err != nil { - logger.Errorf("error converting datatables data parameter from string to int: %v", err) + logger.Warnf("error converting datatables draw parameter from string to int: %v", err) return nil, err } start, err := strconv.ParseUint(q.Get("start"), 10, 64) if err != nil { - logger.Errorf("error converting datatables start parameter from string to int: %v", err) + logger.Warnf("error converting datatables start parameter from string to int: %v", err) return nil, err } + if start > 10000 { + // limit offset to 10000, otherwise the query will be too slow + start = 10000 + } length, err := strconv.ParseInt(q.Get("length"), 10, 64) if err != nil { - logger.Errorf("error converting datatables length parameter from string to int: %v", err) + logger.Warnf("error converting datatables length parameter from string to int: %v", err) return nil, err } if length < 0 { @@ -229,8 +233,8 @@ func ValidatorsData(w http.ResponseWriter, r *http.Request) { dataQuery, err := parseValidatorsDataQueryParams(r) if err != nil { - logger.Errorf("error parsing query-data: %v", err) - http.Error(w, "Internal server error", http.StatusServiceUnavailable) + logger.Warnf("error parsing query-data: %v", err) + http.Error(w, "Error: Invalid query-data parameter.", http.StatusBadRequest) return } @@ -364,6 +368,13 @@ func ValidatorsData(w http.ResponseWriter, r *http.Request) { countFiltered = countTotal } + if countTotal > 10000 { + countTotal = 10000 + } + if countFiltered > 10000 { + countFiltered = 10000 + } + data := &types.DataTableResponse{ Draw: dataQuery.Draw, RecordsTotal: countTotal, diff --git a/handlers/validators_leaderboard.go b/handlers/validators_leaderboard.go index 64dcbc9b1a..eccc2a6fd1 100644 --- a/handlers/validators_leaderboard.go +++ b/handlers/validators_leaderboard.go @@ -56,20 +56,20 @@ func ValidatorsLeaderboardData(w http.ResponseWriter, r *http.Request) { draw, err := strconv.ParseUint(q.Get("draw"), 10, 64) if err != nil { - logger.Errorf("error converting datatables data parameter from string to int: %v", err) - http.Error(w, "Internal server error", http.StatusServiceUnavailable) + logger.Warnf("error converting datatables draw parameter from string to int: %v", err) + http.Error(w, "Error: Missing or invalid parameter draw", http.StatusBadRequest) return } start, err := strconv.ParseUint(q.Get("start"), 10, 64) if err != nil { - logger.Errorf("error converting datatables start parameter from string to int: %v", err) - http.Error(w, "Internal server error", http.StatusServiceUnavailable) + logger.Warnf("error converting datatables start parameter from string to int: %v", err) + http.Error(w, "Error: Missing or invalid parameter start", http.StatusBadRequest) return } length, err := strconv.ParseUint(q.Get("length"), 10, 64) if err != nil { - logger.Errorf("error converting datatables length parameter from string to int: %v", err) - http.Error(w, "Internal server error", http.StatusServiceUnavailable) + logger.Warnf("error converting datatables length parameter from string to int: %v", err) + http.Error(w, "Error: Missing or invalid parameter length", http.StatusBadRequest) return } if length > 100 { diff --git a/handlers/validators_slashings.go b/handlers/validators_slashings.go index ca152a572f..09a115d388 100644 --- a/handlers/validators_slashings.go +++ b/handlers/validators_slashings.go @@ -34,21 +34,21 @@ func ValidatorsSlashingsData(w http.ResponseWriter, r *http.Request) { draw, err := strconv.ParseUint(q.Get("draw"), 10, 64) if err != nil { - logger.Errorf("error converting datatables data parameter from string to int: %v", err) - http.Error(w, "Internal server error", http.StatusServiceUnavailable) + logger.Warnf("error converting datatables draw parameter from string to int: %v", err) + http.Error(w, "Error: Missing or invalid parameter draw", http.StatusBadRequest) return } start, err := strconv.ParseUint(q.Get("start"), 10, 64) if err != nil { - logger.Errorf("error converting datatables start parameter from string to int: %v", err) - http.Error(w, "Internal server error", http.StatusServiceUnavailable) + logger.Warnf("error converting datatables start parameter from string to int: %v", err) + http.Error(w, "Error: Missing or invalid parameter start", http.StatusBadRequest) return } length, err := strconv.ParseUint(q.Get("length"), 10, 64) if err != nil { - logger.Errorf("error converting datatables length parameter from string to int: %v", err) - http.Error(w, "Internal server error", http.StatusServiceUnavailable) + logger.Warnf("error converting datatables length parameter from string to int: %v", err) + http.Error(w, "Error: Missing or invalid parameter length", http.StatusBadRequest) return } if length > 100 { diff --git a/handlers/validators_streakleaderboard.go b/handlers/validators_streakleaderboard.go deleted file mode 100644 index 704d6e4d68..0000000000 --- a/handlers/validators_streakleaderboard.go +++ /dev/null @@ -1,197 +0,0 @@ -package handlers - -import ( - "encoding/json" - "eth2-exporter/db" - "eth2-exporter/templates" - "eth2-exporter/types" - "eth2-exporter/utils" - "fmt" - "net/http" - "strconv" - "strings" -) - -// ValidatorsStreaksLeaderboard returns the attestation-streak-leaderboard using a go template -func ValidatorsStreakLeaderboard(w http.ResponseWriter, r *http.Request) { - templateFiles := append(layoutTemplateFiles, "validators_streakleaderboard.html") - var validatorsStreakLeaderboardTemplate = templates.GetTemplate(templateFiles...) - - w.Header().Set("Content-Type", "text/html") - - data := InitPageData(w, r, "validators", "/validators/streaksleaderboard", "Validator Streaks Leaderboard", templateFiles) - - if handleTemplateError(w, r, "validators_streakLeaderboard.go", "ValidatorsStreakLeaderboard", "", validatorsStreakLeaderboardTemplate.ExecuteTemplate(w, "layout", data)) != nil { - return // an error has occurred and was processed - } -} - -// ValidatorsStreakLeaderboardData returns the leaderboard of attestation-streaks -func ValidatorsStreakLeaderboardData(w http.ResponseWriter, r *http.Request) { - - w.Header().Set("Content-Type", "application/json") - - q := r.URL.Query() - - search := strings.Replace(q.Get("search[value]"), "0x", "", -1) - if len(search) > 128 { - search = search[:128] - } - - draw, err := strconv.ParseUint(q.Get("draw"), 10, 64) - if err != nil { - logger.Errorf("error converting datatables data parameter from string to int: %v", err) - http.Error(w, "Internal server error", http.StatusServiceUnavailable) - return - } - start, err := strconv.ParseUint(q.Get("start"), 10, 64) - if err != nil { - logger.Errorf("error converting datatables start parameter from string to int: %v", err) - http.Error(w, "Internal server error", http.StatusServiceUnavailable) - return - } - length, err := strconv.ParseUint(q.Get("length"), 10, 64) - if err != nil { - logger.Errorf("error converting datatables length parameter from string to int: %v", err) - http.Error(w, "Internal server error", http.StatusServiceUnavailable) - return - } - if length > 100 { - length = 100 - } - - orderColumn := q.Get("order[0][column]") - orderByMap := map[string]string{ - "1": "cs.rank", - "4": "ls.rank", - } - orderBy, exists := orderByMap[orderColumn] - if !exists { - orderBy = "ls.rank" - } - - orderDir := q.Get("order[0][dir]") - if orderDir != "desc" && orderDir != "asc" { - orderDir = "asc" - } - - var totalCount uint64 - - var sqlData []struct { - Totalcount uint64 - Validatorindex uint64 - Name string - Lrank int - Lstart uint64 - Llength int - Crank int - Cstart uint64 - Clength int - } - - if search == "" { - err = db.ReaderDb.Select(&sqlData, ` - with - longeststreaks as ( - select validatorindex, start, length, rank() over(order by length desc) - from validator_attestation_streaks where longest = 't' and status = 1 - ), - currentstreaks as ( - select validatorindex, start, length, rank() over(order by length desc) - from validator_attestation_streaks where current = 't' and status = 1 - ) - select - v.validatorindex, - coalesce(vn.name, '') as name, - cnt.totalcount, - coalesce(ls.rank, 0) lrank, - coalesce(ls.start, 0) lstart, - coalesce(ls.length, 0) llength, - coalesce(cs.rank, 0) crank, - coalesce(cs.start, 0) cstart, - coalesce(cs.length, 0) clength - from longeststreaks ls - inner join validators v on ls.validatorindex = v.validatorindex - left join currentstreaks cs on cs.validatorindex = v.validatorindex - left join validator_names vn on v.pubkey = vn.publickey - left join (select count(*) from longeststreaks) cnt(totalcount) on true - order by `+orderBy+` `+orderDir+` limit $1 offset $2`, length, start) - } else { - err = db.ReaderDb.Select(&sqlData, ` - with - matched_validators as ( - select v.validatorindex, v.pubkey, coalesce(vn.name,'') as name - from validators v - left join validator_names vn ON vn.publickey = v.pubkey - where (pubkeyhex like $4 - or cast(v.validatorindex as text) like $3) - or vn.name ilike $3 - ), - longeststreaks as ( - select validatorindex, start, length, rank() over(order by length desc) - from validator_attestation_streaks where longest = 't' and status = 1 - ), - currentstreaks as ( - select validatorindex, start, length, rank() over(order by length desc) - from validator_attestation_streaks where current = 't' and status = 1 - ) - select - v.validatorindex, - coalesce(vn.name, '') as name, - cnt.totalcount, - coalesce(ls.rank, 0) lrank, - coalesce(ls.start, 0) lstart, - coalesce(ls.length, 0) llength, - coalesce(cs.rank, 0) crank, - coalesce(cs.start, 0) cstart, - coalesce(cs.length, 0) clength - from longeststreaks ls - inner join matched_validators mv on ls.validatorindex = mv.validatorindex - inner join validators v on ls.validatorindex = v.validatorindex - left join currentstreaks cs on cs.validatorindex = v.validatorindex - left join validator_names vn on v.pubkey = vn.publickey - left join (select count(*) from matched_validators) cnt(totalcount) on true - order by `+orderBy+` `+orderDir+` limit $1 offset $2`, length, start, "%"+search+"%", search+"%") - } - if err != nil { - logger.Errorf("error retrieving streaksData data (search=%v): %v", search != "", err) - http.Error(w, "Internal server error", http.StatusServiceUnavailable) - return - } - if len(sqlData) > 0 { - totalCount = sqlData[0].Totalcount - } - - tableData := make([][]interface{}, len(sqlData)) - for i, d := range sqlData { - tableData[i] = []interface{}{ - utils.FormatValidatorWithName(d.Validatorindex, d.Name), - fmt.Sprintf("%v", d.Crank), - utils.FormatEpoch(d.Cstart), - fmt.Sprintf("%v", d.Clength), - fmt.Sprintf("%v", d.Lrank), - utils.FormatEpoch(d.Lstart), - fmt.Sprintf("%v", d.Llength), - } - // current streak is missed - if d.Crank == 0 { - tableData[i][1] = `-` - tableData[i][2] = `-` - tableData[i][3] = `-` - } - } - - data := &types.DataTableResponse{ - Draw: draw, - RecordsTotal: totalCount, - RecordsFiltered: totalCount, - Data: tableData, - } - - err = json.NewEncoder(w).Encode(data) - if err != nil { - logger.Errorf("error enconding json response for %v route: %v", r.URL.String(), err) - http.Error(w, "Internal server error", http.StatusServiceUnavailable) - return - } -} diff --git a/handlers/vis.go b/handlers/vis.go index 7252c193f2..4393256155 100644 --- a/handlers/vis.go +++ b/handlers/vis.go @@ -7,6 +7,7 @@ import ( "eth2-exporter/types" "eth2-exporter/utils" "fmt" + "math" "net/http" "strconv" "time" @@ -41,6 +42,13 @@ func VisBlocks(w http.ResponseWriter, r *http.Request) { sinceSlot := utils.TimeToSlot(uint64(since - 120)) + // slot in postgres is limited to int + if sinceSlot > math.MaxInt32 { + logger.Warnf("error retrieving block tree data, slot too big: %v", err) + http.Error(w, "Error: Invalid parameter since.", http.StatusBadRequest) + return + } + var chartData []*types.VisChartData err = db.ReaderDb.Select(&chartData, "select slot, blockroot, parentroot, proposer from blocks where slot >= $1 and status in ('1', '2') order by slot desc limit 50;", sinceSlot) diff --git a/handlers/withdrawals.go b/handlers/withdrawals.go index 96ffc164c7..7839335ea0 100644 --- a/handlers/withdrawals.go +++ b/handlers/withdrawals.go @@ -59,21 +59,24 @@ func WithdrawalsData(w http.ResponseWriter, r *http.Request) { draw, err := strconv.ParseUint(q.Get("draw"), 10, 64) if err != nil { - logger.Errorf("error converting datatables data parameter from string to int: %v", err) - http.Error(w, "Internal server error", http.StatusServiceUnavailable) + logger.Warnf("error converting datatables draw parameter from string to int: %v", err) + http.Error(w, "Error: Missing or invalid parameter draw", http.StatusBadRequest) return } start, err := strconv.ParseUint(q.Get("start"), 10, 64) if err != nil { - logger.Errorf("error converting datatables start parameter from string to int: %v", err) - http.Error(w, "Internal server error", http.StatusServiceUnavailable) + logger.Warnf("error converting datatables start parameter from string to int: %v", err) + http.Error(w, "Error: Missing or invalid parameter start", http.StatusBadRequest) return } + if start > db.WithdrawalsQueryLimit { + start = db.WithdrawalsQueryLimit + } length, err := strconv.ParseUint(q.Get("length"), 10, 64) if err != nil { - logger.Errorf("error converting datatables length parameter from string to int: %v", err) - http.Error(w, "Internal server error", http.StatusServiceUnavailable) + logger.Warnf("error converting datatables length parameter from string to int: %v", err) + http.Error(w, "Error: Missing or invalid parameter length", http.StatusBadRequest) return } if length > 100 { @@ -181,6 +184,16 @@ func WithdrawalsTableData(draw uint64, search string, length, start uint64, orde formatCurrency = "Ether" } + var err error + names := make(map[string]string) + for _, v := range withdrawals { + names[string(v.Address)] = "" + } + names, _, err = db.BigtableClient.GetAddressesNamesArMetadata(&names, nil) + if err != nil { + return nil, err + } + tableData := make([][]interface{}, len(withdrawals)) for i, w := range withdrawals { tableData[i] = []interface{}{ @@ -189,11 +202,18 @@ func WithdrawalsTableData(draw uint64, search string, length, start uint64, orde template.HTML(fmt.Sprintf("%v", w.Index)), template.HTML(fmt.Sprintf("%v", utils.FormatValidator(w.ValidatorIndex))), template.HTML(fmt.Sprintf("%v", utils.FormatTimestamp(utils.SlotToTime(w.Slot).Unix()))), - template.HTML(fmt.Sprintf("%v", utils.FormatAddress(w.Address, nil, "", false, false, true))), + template.HTML(fmt.Sprintf("%v", utils.FormatAddressWithLimits(w.Address, names[string(w.Address)], false, "address", visibleDigitsForHash+5, 18, true))), template.HTML(fmt.Sprintf("%v", utils.FormatAmount(new(big.Int).Mul(new(big.Int).SetUint64(w.Amount), big.NewInt(1e9)), formatCurrency, 6))), } } + if filteredCount > db.WithdrawalsQueryLimit { + filteredCount = db.WithdrawalsQueryLimit + } + if withdrawalCount > db.WithdrawalsQueryLimit { + withdrawalCount = db.WithdrawalsQueryLimit + } + data := &types.DataTableResponse{ Draw: draw, RecordsTotal: withdrawalCount, @@ -214,20 +234,24 @@ func BLSChangeData(w http.ResponseWriter, r *http.Request) { draw, err := strconv.ParseUint(q.Get("draw"), 10, 64) if err != nil { - logger.Errorf("error converting datatables data parameter from string to int: %v", err) - http.Error(w, "Internal server error", http.StatusServiceUnavailable) + logger.Warnf("error converting datatables draw parameter from string to int: %v", err) + http.Error(w, "Error: Missing or invalid parameter draw", http.StatusBadRequest) return } start, err := strconv.ParseUint(q.Get("start"), 10, 64) if err != nil { - logger.Errorf("error converting datatables start parameter from string to int: %v", err) - http.Error(w, "Internal server error", http.StatusServiceUnavailable) + logger.Warnf("error converting datatables start parameter from string to int: %v", err) + http.Error(w, "Error: Missing or invalid parameter start", http.StatusBadRequest) return } + if start > db.BlsChangeQueryLimit { + // limit offset to 10000, otherwise the query will be too slow + start = db.BlsChangeQueryLimit + } length, err := strconv.ParseUint(q.Get("length"), 10, 64) if err != nil { - logger.Errorf("error converting datatables length parameter from string to int: %v", err) - http.Error(w, "Internal server error", http.StatusServiceUnavailable) + logger.Warnf("error converting datatables length parameter from string to int: %v", err) + http.Error(w, "Error: Missing or invalid parameter length", http.StatusBadRequest) return } if length > 100 { @@ -326,6 +350,16 @@ func BLSTableData(draw uint64, search string, length, start uint64, orderBy, ord filteredCount = totalCount } + var err error + names := make(map[string]string) + for _, v := range blsChange { + names[string(v.Address)] = "" + } + names, _, err = db.BigtableClient.GetAddressesNamesArMetadata(&names, nil) + if err != nil { + return nil, err + } + tableData := make([][]interface{}, len(blsChange)) for i, bls := range blsChange { tableData[i] = []interface{}{ @@ -334,10 +368,17 @@ func BLSTableData(draw uint64, search string, length, start uint64, orderBy, ord template.HTML(fmt.Sprintf("%v", utils.FormatValidator(bls.Validatorindex))), template.HTML(fmt.Sprintf("%v", utils.FormatHashWithCopy(bls.Signature))), template.HTML(fmt.Sprintf("%v", utils.FormatHashWithCopy(bls.BlsPubkey))), - template.HTML(fmt.Sprintf("%v", utils.FormatAddress(bls.Address, nil, "", false, false, true))), + template.HTML(fmt.Sprintf("%v", utils.FormatAddressWithLimits(bls.Address, names[string(bls.Address)], false, "address", visibleDigitsForHash+5, 18, true))), } } + if totalCount > db.BlsChangeQueryLimit { + totalCount = db.BlsChangeQueryLimit + } + if filteredCount > db.BlsChangeQueryLimit { + filteredCount = db.BlsChangeQueryLimit + } + data := &types.DataTableResponse{ Draw: draw, RecordsTotal: totalCount, diff --git a/local-deployment/README.md b/local-deployment/README.md new file mode 100644 index 0000000000..a672f27316 --- /dev/null +++ b/local-deployment/README.md @@ -0,0 +1,76 @@ +This guide outlines how to deploy the explorer using a local lh-geth testnet. Utilized postgres, redis and little_bigtable as data storage + +# Install docker +``` +sudo apt update +sudo apt-get install ca-certificates curl gnupg +sudo install -m 0755 -d /etc/apt/keyrings +curl -fsSL https://download.docker.com/linux/debian/gpg | sudo gpg --dearmor -o /etc/apt/keyrings/docker.gpg +sudo chmod a+r /etc/apt/keyrings/docker.gpg +echo \ + "deb [arch="$(dpkg --print-architecture)" signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/debian \ + "$(. /etc/os-release && echo "$VERSION_CODENAME")" stable" | \ + sudo tee /etc/apt/sources.list.d/docker.list > /dev/null +sudo apt-get update +sudo apt-get install docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin +sudo usermod -aG docker $USER +``` +# Install kurtosis-cli +``` +echo "deb [trusted=yes] https://apt.fury.io/kurtosis-tech/ /" | sudo tee /etc/apt/sources.list.d/kurtosis.list +sudo apt update +sudo apt install kurtosis-cli +``` +# Install golang +``` +wget https://go.dev/dl/go1.20.7.linux-amd64.tar.gz +sudo rm -rf /usr/local/go && sudo tar -C /usr/local -xzf go1.20.7.linux-amd64.tar.gz +``` +Add the golang binaries to the path by adding the following lines to your ~/.profile file and then logout & login again +``` +export PATH=$PATH:/usr/local/go/bin +export PATH=$PATH:$HOME/go/bin +``` +# Clone the explorer repository +``` +cd ~/ +git clone https://github.com/gobitfly/eth2-beaconchain-explorer.git +cd eth2-beaconchain-explorer +``` +# Build the explorer binaries +``` +sudo apt install build-essential +make all +``` +# Start postgres, redis, little_bigtable & the eth test network +``` +cd ~/eth2-beaconchain-explorer/local-deployment/ +kurtosis clean -a && kurtosis run --enclave my-testnet . "$(cat network-params.json)" +``` +# Generate the explorer config file for the deployed testnet +``` +cd ~/eth2-beaconchain-explorer/local-deployment/ +bash provision-explorer-config.sh +``` +This will generate a config.yml to be used by the explorer and then create the bigtable & postgres schema + +# Start the explorer modules +``` +cd ~/eth2-beaconchain-explorer/local-deployment/ +docker-compose up -d +``` +You can start / stop the exporter submodules using `docker-compose` + +# Exit validators +Exiting individual validators can be done using the provided `exit_validator.sh` script. Requires [https://github.com/wealdtech/ethdo](ethdo) to be available on the path. +``` +bash exit_validators.sh -i validator_index -m "memonic" -b "http://bn_api_host:bn_api_port" +``` + +# Enabling withdrawals +The script `add_withdrawal_address.sh` allows you to create & submit a bls to execution layer address change message in order to enable withdrawals for specific validators.. Requires [/github.com/protolambda/eth2-val-tools](eth2-val-tools) to be available on the path. +``` +go get github.com/protolambda/eth2-val-tools@master +go install github.com/protolambda/eth2-val-tools@master +bash add_withdrawal_address.sh -a "EL address (0x1234...)" -i validator_index -m "memonic" -b "http://bn_api_host:bn_api_port" +``` diff --git a/local-deployment/docker-compose.yml b/local-deployment/docker-compose.yml new file mode 100644 index 0000000000..f6ecc16411 --- /dev/null +++ b/local-deployment/docker-compose.yml @@ -0,0 +1,60 @@ +version: '3' +services: + indexer: + build: + context: ../ + args: + - target=explorer + command: ./explorer -config /app/config.yml + volumes: + - ./config.yml:/app/config.yml + environment: + - INDEXER_ENABLED=true + network_mode: "host" + eth1indexer: + build: + context: ../ + args: + - target=eth1indexer + command: ./eth1indexer -config /app/config.yml -blocks.concurrency 1 -blocks.tracemode 'geth' -data.concurrency 1 --balances.enabled + volumes: + - ./config.yml:/app/config.yml + network_mode: "host" + rewards-exporter: + build: + context: ../ + args: + - target=rewards-exporter + command: ./rewards-exporter -config /app/config.yml + volumes: + - ./config.yml:/app/config.yml + network_mode: "host" + statistics: + build: + context: ../ + args: + - target=stats + command: ./statistics -config /app/config.yml --charts.enabled --graffiti.enabled -validators.enabled + volumes: + - ./config.yml:/app/config.yml + network_mode: "host" + frontend-data-updater: + build: + context: ../ + args: + - target=frontend-data-updater + command: ./frontend-data-updater -config /app/config.yml + volumes: + - ./config.yml:/app/config.yml + network_mode: "host" + frontend: + build: + context: ../ + args: + - target=explorer + command: ./explorer -config /app/config.yml + volumes: + - ./config.yml:/app/config.yml + environment: + - FRONTEND_ENABLED=true + network_mode: "host" \ No newline at end of file diff --git a/local-deployment/explorer-config-template.yml b/local-deployment/explorer-config-template.yml new file mode 100644 index 0000000000..6b35d27f16 --- /dev/null +++ b/local-deployment/explorer-config-template.yml @@ -0,0 +1,62 @@ +chain: + configPath: 'node' +readerDatabase: + name: db + host: {{.DBHost}} + port: {{.DBPort}} + user: postgres + password: "pass" +writerDatabase: + name: db + host: {{.DBHost}} + port: {{.DBPort}} + user: postgres + password: "pass" +bigtable: + project: explorer + instance: explorer + emulator: true + emulatorHost: {{.LBTHost}} + emulatorPort: {{.LBTPort}} +eth1ErigonEndpoint: '{{.ELNodeEndpoint}}' +eth1GethEndpoint: '{{.ELNodeEndpoint}}' +redisCacheEndpoint: '{{.RedisEndpoint}}' +tieredCacheProvider: 'redis' +frontend: + siteDomain: "localhost:8080" + siteName: 'Open Source Ethereum (ETH) Testnet Explorer' # Name of the site, displayed in the title tag + siteSubtitle: "Showing a local testnet." + server: + host: '0.0.0.0' # Address to listen on + port: '8080' # Port to listen on + readerDatabase: + name: db + host: {{.DBHost}} + port: {{.DBPort}} + user: postgres + password: "pass" + writerDatabase: + name: db + host: {{.DBHost}} + port: {{.DBPort}} + user: postgres + password: "pass" + sessionSecret: "11111111111111111111111111111111" + jwtSigningSecret: "1111111111111111111111111111111111111111111111111111111111111111" + jwtIssuer: "localhost" + jwtValidityInMinutes: 30 + maxMailsPerEmailPerDay: 10 + mail: + mailgun: + sender: no-reply@localhost + domain: mg.localhost + privateKey: "key-11111111111111111111111111111111" + csrfAuthKey: '1111111111111111111111111111111111111111111111111111111111111111' +indexer: + # fullIndexOnStartup: false # Perform a one time full db index on startup + # indexMissingEpochsOnStartup: true # Check for missing epochs and export them after startup + node: + host: '{{.CLNodeHost}}' + port: '{{.CLNodePort}}' + type: lighthouse + eth1DepositContractFirstBlock: 0 diff --git a/local-deployment/kurtosis.yml b/local-deployment/kurtosis.yml new file mode 100644 index 0000000000..b9e2c298c7 --- /dev/null +++ b/local-deployment/kurtosis.yml @@ -0,0 +1 @@ +name: "github.com/gobitfly/eth2-beaconchain-explorer/local-testnet" \ No newline at end of file diff --git a/local-deployment/main.star b/local-deployment/main.star new file mode 100644 index 0000000000..ad39062bcd --- /dev/null +++ b/local-deployment/main.star @@ -0,0 +1,77 @@ +parse_input = import_module("github.com/kurtosis-tech/eth2-package/src/package_io/parse_input.star") +eth_network_module = import_module("github.com/kurtosis-tech/eth-network-package/main.star") +transaction_spammer = import_module("github.com/kurtosis-tech/eth2-package/src/transaction_spammer/transaction_spammer.star") +genesis_constants = import_module("github.com/kurtosis-tech/eth-network-package/src/prelaunch_data_generator/genesis_constants/genesis_constants.star") +shared_utils = import_module("github.com/kurtosis-tech/eth2-package/src/shared_utils/shared_utils.star") + +POSTGRES_PORT_ID = "postgres" +POSTGRES_DB = "db" +POSTGRES_USER = "postgres" +POSTGRES_PASSWORD = "pass" + +REDIS_PORT_ID = "redis" + +LITTLE_BIGTABLE_PORT_ID = "littlebigtable" + +EXPLORER_CONFIG_FILENAME = "config.yml" + +def run(plan, args): + db_services = plan.add_services( + configs={ + # Add a Postgres server + "postgres": ServiceConfig( + image = "postgres:15.2-alpine", + ports = { + POSTGRES_PORT_ID: PortSpec(5432, application_protocol = "postgresql"), + }, + env_vars = { + "POSTGRES_DB": POSTGRES_DB, + "POSTGRES_USER": POSTGRES_USER, + "POSTGRES_PASSWORD": POSTGRES_PASSWORD, + }, + ), + # Add a Redis server + "redis": ServiceConfig( + image = "redis:7", + ports = { + REDIS_PORT_ID: PortSpec(6379, application_protocol = "tcp"), + }, + ), + # Add a Bigtable Emulator server + "littlebigtable": ServiceConfig( + image = "gobitfly/little_bigtable:latest", + ports = { + LITTLE_BIGTABLE_PORT_ID: PortSpec(9000, application_protocol = "tcp"), + }, + ), + } + ) + + # Spin up a local ethereum testnet + all_participants, cl_genesis_timestamp, genesis_validators_root = eth_network_module.run(plan, args) + + all_el_client_contexts = [] + all_cl_client_contexts = [] + for participant in all_participants: + all_el_client_contexts.append(participant.el_client_context) + all_cl_client_contexts.append(participant.cl_client_context) + + + if args["start_tx_spammer"]: + plan.print("Launching transaction spammer") + transaction_spammer.launch_transaction_spammer(plan, genesis_constants.PRE_FUNDED_ACCOUNTS, all_el_client_contexts[0]) + plan.print("Succesfully launched transaction spammer") + + +def new_config_template_data(cl_node_info, el_uri, lbt_host, lbt_port, db_host, db_port, redis_uri): + return { + "CLNodeHost": cl_node_info.ip_addr, + "CLNodePort": cl_node_info.http_port_num, + "ELNodeEndpoint": el_uri, + "LBTHost": lbt_host, + "LBTPort": lbt_port, + "DBHost": db_host, + "DBPort": db_port, + "RedisEndpoint": redis_uri, + + } \ No newline at end of file diff --git a/local-deployment/network-params.json b/local-deployment/network-params.json new file mode 100644 index 0000000000..13dd9dbe58 --- /dev/null +++ b/local-deployment/network-params.json @@ -0,0 +1,30 @@ +{ + "participants": [ + { + "el_client_type": "geth", + "el_client_image": "", + "el_client_log_level": "", + "cl_client_type": "lighthouse", + "cl_client_image": "", + "cl_client_log_level": "", + "beacon_extra_params": [], + "el_extra_params": ["--http.api='admin,eth,net,debug,txpool' --syncmode full --gcmode archive"], + "validator_extra_params": [], + "builder_network_params": null, + "count": 1 + } + ], + "network_params": { + "preregistered_validator_keys_mnemonic": "giant issue aisle success illegal bike spike question tent bar rely arctic volcano long crawl hungry vocal artwork sniff fantasy very lucky have athlete", + "num_validator_keys_per_node": 64, + "network_id": "3151908", + "deposit_contract_address": "0x4242424242424242424242424242424242424242", + "seconds_per_slot": 4, + "genesis_delay": 10, + "capella_fork_epoch": 3, + "deneb_fork_epoch": 500 + }, + "global_client_log_level": "info", + "start_tx_spammer": false + } + \ No newline at end of file diff --git a/local-deployment/provision-explorer-config.sh b/local-deployment/provision-explorer-config.sh new file mode 100644 index 0000000000..c530a7f317 --- /dev/null +++ b/local-deployment/provision-explorer-config.sh @@ -0,0 +1,101 @@ +#! /bin/bash +CL_PORT=$(kurtosis enclave inspect my-testnet | grep 4000/tcp | tr -s ' ' | cut -d " " -f 6 | sed -e 's/http\:\/\/127.0.0.1\://' | head -n 1) +echo "CL Node port is $CL_PORT" + +EL_PORT=$(kurtosis enclave inspect my-testnet | grep 8545/tcp | tr -s ' ' | cut -d " " -f 5 | sed -e 's/127.0.0.1\://' | head -n 1) +echo "EL Node port is $EL_PORT" + +REDIS_PORT=$(kurtosis enclave inspect my-testnet | grep 6379/tcp | tr -s ' ' | cut -d " " -f 6 | sed -e 's/tcp\:\/\/127.0.0.1\://' | head -n 1) +echo "Redis port is $REDIS_PORT" + +POSTGRES_PORT=$(kurtosis enclave inspect my-testnet | grep 5432/tcp | tr -s ' ' | cut -d " " -f 6 | sed -e 's/postgresql\:\/\/127.0.0.1\://' | head -n 1) +echo "Postgres port is $POSTGRES_PORT" + +LBT_PORT=$(kurtosis enclave inspect my-testnet | grep 9000/tcp | tr -s ' ' | cut -d " " -f 6 | sed -e 's/tcp\:\/\/127.0.0.1\://' | tail -n 1) +echo "Little bigtable port is $LBT_PORT" + +touch config.yml + +cat >config.yml < "/tmp/set_withdrawal_addr/change_operations.json" +cat /tmp/set_withdrawal_addr/change_operations.json +echo "publishing bls-el change message" +curl -X POST $bn_endpoint/eth/v1/beacon/pool/bls_to_execution_changes \ + -H "Content-Type: application/json" \ + --data-binary "@/tmp/set_withdrawal_addr/change_operations.json" +rm -rf /tmp/set_withdrawal_addr \ No newline at end of file diff --git a/local-deployment/scripts/deposit_validator.sh b/local-deployment/scripts/deposit_validator.sh new file mode 100644 index 0000000000..f6f4ca4bc1 --- /dev/null +++ b/local-deployment/scripts/deposit_validator.sh @@ -0,0 +1,49 @@ +#! /bin/bash +set -e + +clean_up () { + ARG=$? + rm -rf /tmp/deposit + exit $ARG +} +trap clean_up EXIT + +while getopts a:b:e:i:m: flag +do + case "${flag}" in + b) bn_endpoint=${OPTARG};; + e) el_endpoint=${OPTARG};; + i) index=${OPTARG};; + m) mnemonic=${OPTARG};; + esac +done + +echo "Validator Index: $index"; +echo "Mnemonic: $mnemonic"; +echo "BN Endpoint: $bn_endpoint"; +echo "EL Endpoint: $el_endpoint"; + +mkdir -p /tmp/deposit +deposit_path="m/44'/60'/0'/0/3" +privatekey="ef5177cd0b6b21c87db5a0bf35d4084a8a57a9d6a064f86d51ac85f2b873a4e2" +publickey="0x878705ba3f8Bc32FCf7F4CAa1A35E72AF65CF766" +fork_version=$(curl -s $bn_endpoint/eth/v1/beacon/genesis | jq -r '.data.genesis_fork_version') +deposit_contract_address=$(curl -s $bn_endpoint/eth/v1/config/spec | jq -r '.data.DEPOSIT_CONTRACT_ADDRESS') +eth2-val-tools deposit-data --source-min=192 --source-max=200 --amount=32000000000 --fork-version=$fork_version --withdrawals-mnemonic="$mnemonic" --validators-mnemonic="$mnemonic" > /tmp/deposit/deposits_0-9.txt +while read x; do + account_name="$(echo "$x" | jq '.account')" + pubkey="$(echo "$x" | jq '.pubkey')" + echo "Sending deposit for validator $account_name $pubkey" + ethereal beacon deposit \ + --allow-unknown-contract=true \ + --address="$deposit_contract_address" \ + --connection=$el_endpoint \ + --data="$x" \ + --value="32000000000" \ + --from="$publickey" \ + --privatekey="$privatekey" + echo "Sent deposit for validator $account_name $pubkey" + sleep 3 +done < /tmp/deposit/deposits_0-9.txt +exit; +rm -rf /tmp/deposit \ No newline at end of file diff --git a/local-deployment/scripts/exit_validator.sh b/local-deployment/scripts/exit_validator.sh new file mode 100644 index 0000000000..1bb3f0f9c4 --- /dev/null +++ b/local-deployment/scripts/exit_validator.sh @@ -0,0 +1,33 @@ +#! /bin/bash +set -e + +clean_up () { + ARG=$? + rm -rf /tmp/full_withdrawal + exit $ARG +} +trap clean_up EXIT + +while getopts b:i:m: flag +do + case "${flag}" in + b) bn_endpoint=${OPTARG};; + i) index=${OPTARG};; + m) mnemonic=${OPTARG};; + esac +done +echo "Validator Index: $index"; +echo "Mnemonic: $mnemonic"; +echo "BN Endpoint: $bn_endpoint"; + +mkdir -p /tmp/full_withdrawal +echo "creating wallet" +ethdo wallet create --base-dir=/tmp/full_withdrawal --type=hd --wallet=withdrawal-validators --mnemonic="$mnemonic" --wallet-passphrase="superSecure" --allow-weak-passphrases +echo "deriving account wallet" +ethdo account create --base-dir=/tmp/full_withdrawal --account=withdrawal-validators/$index --wallet-passphrase="superSecure" --passphrase="superSecure" --allow-weak-passphrases --path="m/12381/3600/$index/0/0" +echo "creating exit message" +ethdo validator exit --base-dir=/tmp/full_withdrawal --json --account=withdrawal-validators/$index --passphrase="superSecure" --connection=$bn_endpoint > /tmp/full_withdrawal/withdrawal-$index.json +echo "submitting exit message" +ethdo validator exit --signed-operations=$(cat /tmp/full_withdrawal/withdrawal-$index.json) --connection=$bn_endpoint +ethdo wallet delete --base-dir=/tmp/full_withdrawal --wallet=withdrawal-validators +rm -rf /tmp/full_withdrawal \ No newline at end of file diff --git a/notify/firebase.go b/notify/firebase.go index bf1a642671..55e9400378 100644 --- a/notify/firebase.go +++ b/notify/firebase.go @@ -3,6 +3,7 @@ package notify import ( "context" "eth2-exporter/utils" + "fmt" "strings" "time" @@ -33,7 +34,18 @@ func SendPushBatch(messages []*messaging.Message) error { } ctx := context.Background() - opt := option.WithCredentialsFile(credentialsPath) + var opt option.ClientOption + + if strings.HasPrefix(credentialsPath, "projects/") { + x, err := utils.AccessSecretVersion(credentialsPath) + if err != nil { + return fmt.Errorf("error getting firebase config from secret store: %v", err) + } + opt = option.WithCredentialsJSON([]byte(*x)) + } else { + opt = option.WithCredentialsFile(credentialsPath) + } + app, err := firebase.NewApp(context.Background(), nil, opt) if err != nil { logger.Errorf("error initializing app: %v", err) diff --git a/rpc/client.go b/rpc/client.go new file mode 100644 index 0000000000..adb62de71e --- /dev/null +++ b/rpc/client.go @@ -0,0 +1,3 @@ +package rpc + +var CurrentClient *LighthouseClient diff --git a/rpc/erigon.go b/rpc/erigon.go index 49a32abfb4..9b2b08ce73 100644 --- a/rpc/erigon.go +++ b/rpc/erigon.go @@ -73,7 +73,7 @@ func (client *ErigonClient) GetRPCClient() *geth_rpc.Client { return client.rpcClient } -func (client *ErigonClient) GetBlock(number int64) (*types.Eth1Block, *types.GetBlockTimings, error) { +func (client *ErigonClient) GetBlock(number int64, traceMode string) (*types.Eth1Block, *types.GetBlockTimings, error) { ctx, cancel := context.WithTimeout(context.Background(), time.Second*10) defer cancel() @@ -207,10 +207,75 @@ func (client *ErigonClient) GetBlock(number int64) (*types.Eth1Block, *types.Get g := new(errgroup.Group) g.Go(func() error { - traces, err := client.TraceParity(block.NumberU64()) + if block.NumberU64() == 0 { // genesis block is not traceable + return nil + } - if err != nil { - logger.Errorf("error tracing block via parity style traces (%v), %v: %v", block.Number(), block.Hash(), err) + var traceError error + if traceMode == "parity" || traceMode == "parity/geth" { + traces, err := client.TraceParity(block.NumberU64()) + + if err != nil { + if traceMode == "parity" { + return fmt.Errorf("error tracing block via parity style traces (%v), %v: %v", block.Number(), block.Hash(), err) + } else { + logger.Errorf("error tracing block via parity style traces (%v), %v: %v", block.Number(), block.Hash(), err) + + } + traceError = err + } else { + for _, trace := range traces { + if trace.Type == "reward" { + continue + } + + if trace.TransactionHash == "" { + continue + } + + if trace.TransactionPosition >= len(c.Transactions) { + return fmt.Errorf("error transaction position %v out of range", trace.TransactionPosition) + } + + if trace.Error == "" { + c.Transactions[trace.TransactionPosition].Status = 1 + } else { + c.Transactions[trace.TransactionPosition].Status = 0 + c.Transactions[trace.TransactionPosition].ErrorMsg = trace.Error + } + + tracePb := &types.Eth1InternalTransaction{ + Type: trace.Type, + Path: fmt.Sprint(trace.TraceAddress), + } + + if tracePb.Type == "call" { + tracePb.Type = trace.Action.CallType + } + + if trace.Type == "create" { + tracePb.From = common.FromHex(trace.Action.From) + tracePb.To = common.FromHex(trace.Result.Address) + tracePb.Value = common.FromHex(trace.Action.Value) + } else if trace.Type == "suicide" { + tracePb.From = common.FromHex(trace.Action.Address) + tracePb.To = common.FromHex(trace.Action.RefundAddress) + tracePb.Value = common.FromHex(trace.Action.Balance) + } else if trace.Type == "call" { + tracePb.From = common.FromHex(trace.Action.From) + tracePb.To = common.FromHex(trace.Action.To) + tracePb.Value = common.FromHex(trace.Action.Value) + } else { + spew.Dump(trace) + logrus.Fatalf("unknown trace type %v in tx %v", trace.Type, trace.TransactionHash) + } + + c.Transactions[trace.TransactionPosition].Itx = append(c.Transactions[trace.TransactionPosition].Itx, tracePb) + } + } + } + + if traceMode == "geth" || (traceError != nil && traceMode == "parity/geth") { gethTraceData, err := client.TraceGeth(block.Hash()) @@ -218,7 +283,7 @@ func (client *ErigonClient) GetBlock(number int64) (*types.Eth1Block, *types.Get return fmt.Errorf("error tracing block via geth style traces (%v), %v: %v", block.Number(), block.Hash(), err) } - logger.Infof("retrieved %v calls via geth", len(gethTraceData)) + // logger.Infof("retrieved %v calls via geth", len(gethTraceData)) for _, trace := range gethTraceData { if trace.Error == "" { @@ -252,7 +317,7 @@ func (client *ErigonClient) GetBlock(number int64) (*types.Eth1Block, *types.Get logrus.Fatalf("unknown trace type %v in tx %v", trace.Type, trace.TransactionPosition) } - logger.Infof("appending trace %v to tx %x from %v to %v value %v", trace.TransactionPosition, c.Transactions[trace.TransactionPosition].Hash, trace.From, trace.To, trace.Value) + logger.Tracef("appending trace %v to tx %x from %v to %v value %v", trace.TransactionPosition, c.Transactions[trace.TransactionPosition].Hash, trace.From, trace.To, trace.Value) c.Transactions[trace.TransactionPosition].Itx = append(c.Transactions[trace.TransactionPosition].Itx, tracePb) } @@ -261,54 +326,7 @@ func (client *ErigonClient) GetBlock(number int64) (*types.Eth1Block, *types.Get timings.Traces = time.Since(start) // logrus.Infof("retrieved %v traces for %v txs", len(traces), len(c.Transactions)) - for _, trace := range traces { - if trace.Type == "reward" { - continue - } - - if trace.TransactionHash == "" { - continue - } - - if trace.TransactionPosition >= len(c.Transactions) { - return fmt.Errorf("error transaction position %v out of range", trace.TransactionPosition) - } - - if trace.Error == "" { - c.Transactions[trace.TransactionPosition].Status = 1 - } else { - c.Transactions[trace.TransactionPosition].Status = 0 - c.Transactions[trace.TransactionPosition].ErrorMsg = trace.Error - } - - tracePb := &types.Eth1InternalTransaction{ - Type: trace.Type, - Path: fmt.Sprint(trace.TraceAddress), - } - - if tracePb.Type == "call" { - tracePb.Type = trace.Action.CallType - } - if trace.Type == "create" { - tracePb.From = common.FromHex(trace.Action.From) - tracePb.To = common.FromHex(trace.Result.Address) - tracePb.Value = common.FromHex(trace.Action.Value) - } else if trace.Type == "suicide" { - tracePb.From = common.FromHex(trace.Action.Address) - tracePb.To = common.FromHex(trace.Action.RefundAddress) - tracePb.Value = common.FromHex(trace.Action.Balance) - } else if trace.Type == "call" { - tracePb.From = common.FromHex(trace.Action.From) - tracePb.To = common.FromHex(trace.Action.To) - tracePb.Value = common.FromHex(trace.Action.Value) - } else { - spew.Dump(trace) - logrus.Fatalf("unknown trace type %v in tx %v", trace.Type, trace.TransactionHash) - } - - c.Transactions[trace.TransactionPosition].Itx = append(c.Transactions[trace.TransactionPosition].Itx, tracePb) - } return nil }) diff --git a/rpc/lighthouse.go b/rpc/lighthouse.go index 59c81f64b5..f5f8becee6 100644 --- a/rpc/lighthouse.go +++ b/rpc/lighthouse.go @@ -8,7 +8,7 @@ import ( "eth2-exporter/types" "eth2-exporter/utils" "fmt" - "io/ioutil" + "io" "math/big" "net/http" "strconv" @@ -18,10 +18,10 @@ import ( "github.com/donovanhide/eventsource" gtypes "github.com/ethereum/go-ethereum/core/types" + "golang.org/x/sync/errgroup" lru "github.com/hashicorp/golang-lru" "github.com/prysmaticlabs/go-bitfield" - "github.com/sirupsen/logrus" ) // LighthouseLatestHeadEpoch is used to cache the latest head epoch for participation requests @@ -177,7 +177,7 @@ func (lc *LighthouseClient) GetEpochAssignments(epoch uint64) (*types.EpochAssig proposerResp, err := lc.get(fmt.Sprintf("%s/eth/v1/validator/duties/proposer/%d", lc.endpoint, epoch)) if err != nil { - return nil, fmt.Errorf("error retrieving proposer duties: %v", err) + return nil, fmt.Errorf("error retrieving proposer duties for epoch %v: %v", epoch, err) } var parsedProposerResponse StandardProposerDutiesResponse err = json.Unmarshal(proposerResp, &parsedProposerResponse) @@ -258,13 +258,43 @@ func (lc *LighthouseClient) GetEpochAssignments(epoch uint64) (*types.EpochAssig return assignments, nil } +func (lc *LighthouseClient) GetValidatorState(epoch uint64) (*StandardValidatorsResponse, error) { + validatorsResp, err := lc.get(fmt.Sprintf("%s/eth/v1/beacon/states/%d/validators", lc.endpoint, epoch*utils.Config.Chain.ClConfig.SlotsPerEpoch)) + if err != nil && epoch == 0 { + validatorsResp, err = lc.get(fmt.Sprintf("%s/eth/v1/beacon/states/%v/validators", lc.endpoint, "genesis")) + if err != nil { + return nil, fmt.Errorf("error retrieving validators for genesis: %v", err) + } + } else if err != nil { + return nil, fmt.Errorf("error retrieving validators for epoch %v: %v", epoch, err) + } + + parsedValidators := &StandardValidatorsResponse{} + err = json.Unmarshal(validatorsResp, parsedValidators) + if err != nil { + return nil, fmt.Errorf("error parsing epoch validators: %v", err) + } + + return parsedValidators, nil +} + // GetEpochData will get the epoch data from Lighthouse RPC api func (lc *LighthouseClient) GetEpochData(epoch uint64, skipHistoricBalances bool) (*types.EpochData, error) { - wg := &sync.WaitGroup{} + wg := &errgroup.Group{} mux := &sync.Mutex{} - data := &types.EpochData{} + head, err := lc.GetChainHead() + if err != nil { + return nil, fmt.Errorf("error retrieving chain head: %v", err) + } + data := &types.EpochData{ + SyncDuties: make(map[types.Slot]map[types.ValidatorIndex]bool), + AttestationDuties: make(map[types.Slot]map[types.ValidatorIndex][]types.Slot), + } data.Epoch = epoch + if head.FinalizedEpoch >= epoch { + data.Finalized = true + } validatorsResp, err := lc.get(fmt.Sprintf("%s/eth/v1/beacon/states/%d/validators", lc.endpoint, epoch*utils.Config.Chain.ClConfig.SlotsPerEpoch)) if err != nil && epoch == 0 { @@ -300,52 +330,87 @@ func (lc *LighthouseClient) GetEpochData(epoch uint64, skipHistoricBalances bool logger.Printf("retrieved data for %v validators for epoch %v", len(data.Validators), epoch) - wg.Add(1) - go func() { - defer wg.Done() + wg.Go(func() error { var err error data.ValidatorAssignmentes, err = lc.GetEpochAssignments(epoch) if err != nil { - logrus.Errorf("error retrieving assignments for epoch %v: %v", epoch, err) - return + return fmt.Errorf("error retrieving assignments for epoch %v: %v", epoch, err) } - logger.Printf("retrieved validator assignment data for epoch %v", epoch) - }() - wg.Add(1) - go func() { - defer wg.Done() - data.EpochParticipationStats, err = lc.GetValidatorParticipation(epoch) - if err != nil { - if strings.HasSuffix(err.Error(), "can't be retrieved as it hasn't finished yet") { - logger.Warnf("error retrieving epoch participation statistics for epoch %v: %v", epoch, err) - } else { - logger.Errorf("error retrieving epoch participation statistics for epoch %v: %v", epoch, err) + for slot := epoch * utils.Config.Chain.ClConfig.SlotsPerEpoch; slot <= (epoch+1)*utils.Config.Chain.ClConfig.SlotsPerEpoch-1; slot++ { + if data.SyncDuties[types.Slot(slot)] == nil { + data.SyncDuties[types.Slot(slot)] = make(map[types.ValidatorIndex]bool) } - data.EpochParticipationStats = &types.ValidatorParticipation{ - Epoch: epoch, - GlobalParticipationRate: 1.0, - VotedEther: 0, - EligibleEther: 0, + for _, validatorIndex := range data.ValidatorAssignmentes.SyncAssignments { + data.SyncDuties[types.Slot(slot)][types.ValidatorIndex(validatorIndex)] = false } } - }() + + for key, validatorIndex := range data.ValidatorAssignmentes.AttestorAssignments { + keySplit := strings.Split(key, "-") + attestedSlot, err := strconv.ParseUint(keySplit[0], 10, 64) + + if err != nil { + return fmt.Errorf("error parsing attested slot from attestation key: %v", err) + } + + if data.AttestationDuties[types.Slot(attestedSlot)] == nil { + data.AttestationDuties[types.Slot(attestedSlot)] = make(map[types.ValidatorIndex][]types.Slot) + } + + data.AttestationDuties[types.Slot(attestedSlot)][types.ValidatorIndex(validatorIndex)] = []types.Slot{} + + } + logger.Printf("retrieved validator assignment data for epoch %v", epoch) + return nil + }) + + if epoch < head.HeadEpoch { + wg.Go(func() error { + data.EpochParticipationStats, err = lc.GetValidatorParticipation(epoch) + if err != nil { + if strings.HasSuffix(err.Error(), "can't be retrieved as it hasn't finished yet") { // should no longer happen + logger.Warnf("error retrieving epoch participation statistics for epoch %v: %v", epoch, err) + } else { + return fmt.Errorf("error retrieving epoch participation statistics for epoch %v: %v", epoch, err) + } + data.EpochParticipationStats = &types.ValidatorParticipation{ + Epoch: epoch, + GlobalParticipationRate: 1.0, + VotedEther: 0, + EligibleEther: 0, + } + } + return nil + }) + } else { + data.EpochParticipationStats = &types.ValidatorParticipation{ + Epoch: epoch, + GlobalParticipationRate: 1.0, + VotedEther: 0, + EligibleEther: 0, + } + } + + err = wg.Wait() + if err != nil { + return nil, err + } + wg = &errgroup.Group{} // Retrieve all blocks for the epoch data.Blocks = make(map[uint64]map[string]*types.Block) for slot := epoch * utils.Config.Chain.ClConfig.SlotsPerEpoch; slot <= (epoch+1)*utils.Config.Chain.ClConfig.SlotsPerEpoch-1; slot++ { - if slot != 0 && utils.SlotToTime(slot).After(time.Now()) { // don't export slots that have not occured yet + if slot != 0 && slot > head.HeadSlot { // don't export slots that have not occured yet continue } - wg.Add(1) - go func(slot uint64) { - defer wg.Done() + slot := slot + wg.Go(func() error { blocks, err := lc.GetBlocksBySlot(slot) if err != nil { - logger.Errorf("error retrieving blocks for slot %v: %v", slot, err) - return + return fmt.Errorf("error retrieving blocks for slot %v: %v", slot, err) } for _, block := range blocks { @@ -354,11 +419,63 @@ func (lc *LighthouseClient) GetEpochData(epoch uint64, skipHistoricBalances bool data.Blocks[block.Slot] = make(map[string]*types.Block) } data.Blocks[block.Slot][fmt.Sprintf("%x", block.BlockRoot)] = block + + for validator, duty := range block.SyncDuties { + data.SyncDuties[types.Slot(block.Slot)][types.ValidatorIndex(validator)] = duty + } + for validator, attestedSlot := range block.AttestationDuties { + if data.AttestationDuties[types.Slot(attestedSlot)] == nil { + data.AttestationDuties[types.Slot(attestedSlot)] = make(map[types.ValidatorIndex][]types.Slot) + } + if data.AttestationDuties[types.Slot(attestedSlot)][types.ValidatorIndex(validator)] == nil { + data.AttestationDuties[types.Slot(attestedSlot)][types.ValidatorIndex(validator)] = make([]types.Slot, 0, 10) + } + data.AttestationDuties[types.Slot(attestedSlot)][types.ValidatorIndex(validator)] = append(data.AttestationDuties[types.Slot(attestedSlot)][types.ValidatorIndex(validator)], types.Slot(block.Slot)) + } + mux.Unlock() + } + return nil + }) + } + + // we need future blocks to properly tracke fullfilled attestation duties + data.FutureBlocks = make(map[uint64]map[string]*types.Block) + for slot := (epoch + 1) * utils.Config.Chain.ClConfig.SlotsPerEpoch; slot <= (epoch+2)*utils.Config.Chain.ClConfig.SlotsPerEpoch-1; slot++ { + if slot != 0 && slot > head.HeadSlot { // don't export slots that have not occured yet + continue + } + slot := slot + wg.Go(func() error { + blocks, err := lc.GetBlocksBySlot(slot) + + if err != nil { + return fmt.Errorf("error retrieving blocks for slot %v: %v", slot, err) + } + + for _, block := range blocks { + mux.Lock() + if data.FutureBlocks[block.Slot] == nil { + data.FutureBlocks[block.Slot] = make(map[string]*types.Block) + } + data.FutureBlocks[block.Slot][fmt.Sprintf("%x", block.BlockRoot)] = block + + // fill out performed attestation duties + for validator, attestedSlot := range block.AttestationDuties { + if attestedSlot < types.Slot((epoch+1)*utils.Config.Chain.ClConfig.SlotsPerEpoch) { + data.AttestationDuties[types.Slot(attestedSlot)][types.ValidatorIndex(validator)] = append(data.AttestationDuties[types.Slot(attestedSlot)][types.ValidatorIndex(validator)], types.Slot(block.Slot)) + } + } mux.Unlock() } - }(slot) + return nil + }) + } + + err = wg.Wait() + if err != nil { + return nil, err } - wg.Wait() + logger.Printf("retrieved %v blocks for epoch %v", len(data.Blocks), epoch) if data.ValidatorAssignmentes == nil { @@ -404,6 +521,12 @@ func (lc *LighthouseClient) GetEpochData(epoch uint64, skipHistoricBalances bool } } + // for validator, duty := range data.AttestationDuties { + // for slot, inclusions := range duty { + // logger.Infof("validator %v has attested for slot %v in slots %v", validator, slot, inclusions) + // } + // } + return data, nil } @@ -566,6 +689,8 @@ func (lc *LighthouseClient) blockFromResponse(parsedHeaders *StandardBeaconHeade SignedBLSToExecutionChange: make([]*types.SignedBLSToExecutionChange, len(parsedBlock.Message.Body.SignedBLSToExecutionChange)), BlobKZGCommitments: make([][]byte, len(parsedBlock.Message.Body.BlobKZGCommitments)), BlobKZGProofs: make([][]byte, len(parsedBlock.Message.Body.BlobKZGCommitments)), + AttestationDuties: make(map[types.ValidatorIndex]types.Slot), + SyncDuties: make(map[types.ValidatorIndex]bool), } for i, c := range parsedBlock.Message.Body.BlobKZGCommitments { @@ -606,6 +731,16 @@ func (lc *LighthouseClient) blockFromResponse(parsedHeaders *StandardBeaconHeade SyncAggregateParticipation: syncCommitteeParticipation(bits), SyncCommitteeSignature: utils.MustParseHex(agg.SyncCommitteeSignature), } + + // fill out performed sync duties + bitLen := len(block.SyncAggregate.SyncCommitteeBits) * 8 + valLen := len(block.SyncAggregate.SyncCommitteeValidators) + if bitLen < valLen { + return nil, fmt.Errorf("error getting sync_committee participants: bitLen != valLen: %v != %v", bitLen, valLen) + } + for i, valIndex := range block.SyncAggregate.SyncCommitteeValidators { + block.SyncDuties[types.ValidatorIndex(valIndex)] = utils.BitAtVector(block.SyncAggregate.SyncCommitteeBits, i) + } } if payload := parsedBlock.Message.Body.ExecutionPayload; payload != nil && !bytes.Equal(payload.ParentHash, make([]byte, 32)) { @@ -774,6 +909,8 @@ func (lc *LighthouseClient) blockFromResponse(parsedHeaders *StandardBeaconHeade logger.Errorf("error retrieving assigned validator for attestation %v of block %v for slot %v committee index %v member index %v", i, block.Slot, a.Data.Slot, a.Data.CommitteeIndex, i) } a.Attesters = append(a.Attesters, validator) + + block.AttestationDuties[types.ValidatorIndex(validator)] = types.Slot(a.Data.Slot) } } @@ -826,13 +963,15 @@ func syncCommitteeParticipation(bits []byte) float64 { // GetValidatorParticipation will get the validator participation from the Lighthouse RPC api func (lc *LighthouseClient) GetValidatorParticipation(epoch uint64) (*types.ValidatorParticipation, error) { + + head, err := lc.GetChainHead() + if err != nil { + return nil, err + } + if LighthouseLatestHeadEpoch == 0 || epoch >= LighthouseLatestHeadEpoch-1 { // update LighthouseLatestHeadEpoch to make sure we are continuing with the latest data // we need to check when epoch = head and epoch head - 1 so our following logic acts correctly when we are close to the head - head, err := lc.GetChainHead() - if err != nil { - return nil, err - } logger.Infof("Updating LighthouseLatestHeadEpoch to %v", head.HeadEpoch) LighthouseLatestHeadEpoch = head.HeadEpoch } @@ -875,6 +1014,7 @@ func (lc *LighthouseClient) GetValidatorParticipation(epoch uint64) (*types.Vali GlobalParticipationRate: float32(parsedResponse.Data.PreviousEpochTargetAttestingGwei) / float32(parsedResponse.Data.PreviousEpochActiveGwei), VotedEther: uint64(parsedResponse.Data.PreviousEpochTargetAttestingGwei), EligibleEther: uint64(parsedResponse.Data.PreviousEpochActiveGwei), + Finalized: epoch <= head.FinalizedEpoch, } } else { res = &types.ValidatorParticipation{ @@ -882,6 +1022,7 @@ func (lc *LighthouseClient) GetValidatorParticipation(epoch uint64) (*types.Vali GlobalParticipationRate: float32(parsedResponse.Data.CurrentEpochTargetAttestingGwei) / float32(parsedResponse.Data.CurrentEpochActiveGwei), VotedEther: uint64(parsedResponse.Data.CurrentEpochTargetAttestingGwei), EligibleEther: uint64(parsedResponse.Data.CurrentEpochActiveGwei), + Finalized: epoch <= head.FinalizedEpoch, } } return res, nil @@ -934,7 +1075,7 @@ func (lc *LighthouseClient) get(url string) ([]byte, error) { defer resp.Body.Close() - data, err := ioutil.ReadAll(resp.Body) + data, err := io.ReadAll(resp.Body) if resp.StatusCode != http.StatusOK { if resp.StatusCode == http.StatusNotFound { diff --git a/services/charts_updater.go b/services/charts_updater.go index af780fc883..c0e8bc519b 100644 --- a/services/charts_updater.go +++ b/services/charts_updater.go @@ -4,6 +4,7 @@ import ( "eth2-exporter/cache" "eth2-exporter/db" "eth2-exporter/metrics" + "eth2-exporter/rpc" "eth2-exporter/types" "eth2-exporter/utils" "fmt" @@ -74,7 +75,7 @@ func LatestChartsPageData() []*types.ChartsPageDataChart { } func chartsPageDataUpdater(wg *sync.WaitGroup) { - sleepDuration := time.Second * time.Duration(utils.Config.Chain.ClConfig.SecondsPerSlot) + sleepDuration := time.Hour // only update charts once per hour var prevEpoch uint64 firstun := true @@ -599,18 +600,18 @@ func balanceDistributionChartData() (*types.GenericChartData, error) { return nil, fmt.Errorf("chart-data not available pre-genesis") } - balances, err := db.BigtableClient.GetValidatorBalanceHistory([]uint64{}, epoch, epoch) + validators, err := rpc.CurrentClient.GetValidatorState(epoch) if err != nil { return nil, err } - currentBalances := make([]float64, 0, len(balances)) + if validators.Data == nil { + return nil, fmt.Errorf("GetValidatorState returned empty validator set for epoch %v", epoch) + } - for _, balance := range balances { - if len(balance) == 0 { - continue - } - currentBalances = append(currentBalances, float64(balance[0].Balance)/1e9) + currentBalances := make([]float64, 0, len(validators.Data)) + for _, entry := range validators.Data { + currentBalances = append(currentBalances, float64(entry.Balance)/1e9) } bins := int(math.Sqrt(float64(len(currentBalances)))) + 1 @@ -649,18 +650,19 @@ func effectiveBalanceDistributionChartData() (*types.GenericChartData, error) { return nil, fmt.Errorf("chart-data not available pre-genesis") } - balances, err := db.BigtableClient.GetValidatorBalanceHistory([]uint64{}, epoch, epoch) + validators, err := rpc.CurrentClient.GetValidatorState(epoch) if err != nil { return nil, err } - effectiveBalances := make([]float64, 0, len(balances)) + if validators.Data == nil { + return nil, fmt.Errorf("GetValidatorState returned empty validator set for epoch %v", epoch) + } - for _, balance := range balances { - if len(balance) == 0 { - continue - } - effectiveBalances = append(effectiveBalances, float64(balance[0].EffectiveBalance)/1e9) + effectiveBalances := make([]float64, 0, len(validators.Data)) + + for _, entry := range validators.Data { + effectiveBalances = append(effectiveBalances, float64(entry.Validator.EffectiveBalance)/1e9) } bins := int(math.Sqrt(float64(len(effectiveBalances)))) + 1 diff --git a/services/configuration.go b/services/configuration.go index df89548989..1423149236 100644 --- a/services/configuration.go +++ b/services/configuration.go @@ -16,10 +16,12 @@ const ( ConfigurationKeyHardforkName types.ExplorerConfigurationKey = "HardforkName" ) -/*** +/* +** This is the list of possible configurations that can be changed in the explorer administration Per default these values will be taken, overridden by the values from the db, if they exist. -***/ +** +*/ var DefaultExplorerConfiguration types.ExplorerConfigurationMap = types.ExplorerConfigurationMap{ ConfigurationCategorySlotViz: { ConfigurationKeyVisibleFromEpoch: {Value: "0", DataType: "int"}, diff --git a/services/notifications.go b/services/notifications.go index ef8519189d..d0b4063b04 100644 --- a/services/notifications.go +++ b/services/notifications.go @@ -1382,7 +1382,13 @@ func collectAttestationAndOfflineValidatorNotifications(notificationsByUserID ma } // get attestations for all validators for the last 4 epochs - attestations, err := db.BigtableClient.GetValidatorAttestationHistory([]uint64{}, epoch-3, epoch) + + validators, err := db.GetValidatorIndices() + if err != nil { + return err + } + + attestations, err := db.BigtableClient.GetValidatorAttestationHistory(validators, epoch-3, epoch) if err != nil { return fmt.Errorf("error getting validator attestations from bigtable %w", err) } @@ -2330,7 +2336,7 @@ func collectMonitoringMachine( rowKeys := gcp_bigtable.RowList{} for _, data := range allSubscribed { - rowKeys = append(rowKeys, db.GetMachineRowKey(data.UserID, "system", data.MachineName)) + rowKeys = append(rowKeys, db.BigtableClient.GetMachineRowKey(data.UserID, "system", data.MachineName)) } machineDataOfSubscribed, err := db.BigtableClient.GetMachineMetricsForNotifications(rowKeys) @@ -2587,8 +2593,11 @@ func (n *taxReportNotification) GetInfoMarkdown() string { } func collectTaxReportNotificationNotifications(notificationsByUserID map[uint64]map[types.EventName][]types.Notification, eventName types.EventName) error { - lastStatsDay := LatestExportedStatisticDay() + lastStatsDay, err := LatestExportedStatisticDay() + if err != nil { + return err + } //Check that the last day of the month is already exported tNow := time.Now() firstDayOfMonth := time.Date(tNow.Year(), tNow.Month(), 1, 0, 0, 0, 0, time.UTC) @@ -2609,7 +2618,7 @@ func collectTaxReportNotificationNotifications(notificationsByUserID map[uint64] name = utils.Config.Chain.ClConfig.ConfigName + ":" + name } - err := db.FrontendWriterDB.Select(&dbResult, ` + err = db.FrontendWriterDB.Select(&dbResult, ` SELECT us.id, us.user_id, us.created_epoch, us.event_filter, ENCODE(us.unsubscribe_hash, 'hex') as unsubscribe_hash FROM users_subscriptions AS us WHERE us.event_name=$1 AND (us.last_sent_ts < $2 OR (us.last_sent_ts IS NULL AND us.created_ts < $2)); @@ -2790,9 +2799,9 @@ func (n *rocketpoolNotification) GetInfo(includeUrl bool) string { case types.RocketpoolNewClaimRoundStartedEventName: return `A new reward round has started. You can now claim your rewards from the previous round.` case types.RocketpoolCollateralMaxReached: - return `Your RPL collateral has reached your configured threshold at 150%.` + return fmt.Sprintf(`Your RPL collateral has reached your configured threshold at %v%%.`, n.ExtraData) case types.RocketpoolCollateralMinReached: - return `Your RPL collateral has reached your configured threshold at 10%.` + return fmt.Sprintf(`Your RPL collateral has reached your configured threshold at %v%%.`, n.ExtraData) case types.SyncCommitteeSoon: extras := strings.Split(n.ExtraData, "|") if len(extras) != 3 { @@ -2960,7 +2969,7 @@ func collectRocketpoolRPLCollateralNotifications(notificationsByUserID map[uint6 RPLStakeMax BigFloat `db:"max_rpl_stake"` } - events := make([]dbResult, 0) + stakeInfoPerNode := make([]dbResult, 0) batchSize := 5000 dataLen := len(pubkeys) for i := 0; i < dataLen; i += batchSize { @@ -2976,39 +2985,65 @@ func collectRocketpoolRPLCollateralNotifications(notificationsByUserID map[uint6 var partial []dbResult + // filter nodes with no minipools (anymore) because they have min/max stake of 0 + // TODO properly remove notification entry from db err = db.WriterDb.Select(&partial, ` - SELECT address, rpl_stake, min_rpl_stake, max_rpl_stake - FROM rocketpool_nodes WHERE address = ANY($1)`, pq.ByteaArray(keys)) + SELECT address, rpl_stake, min_rpl_stake, max_rpl_stake + FROM rocketpool_nodes + WHERE address = ANY($1) AND min_rpl_stake != 0 AND max_rpl_stake != 0`, pq.ByteaArray(keys)) if err != nil { return err } - events = append(events, partial...) + stakeInfoPerNode = append(stakeInfoPerNode, partial...) } - for _, r := range events { + // factor in network-wide min/max collat ratio. Since LEB8 they are not directly correlated anymore (ratio of bonded to borrowed ETH), so we need either min or max + // however this is dynamic and might be changed in the future; Should extend rocketpool_network_stats to include min/max collateral values! + minRPLCollatRatio := bigFloat(0.1) // bigFloat it to save some memory re-allocations + maxRPLCollatRatio := bigFloat(1.5) + // temporary helper (modifying values in dbResult directly would be bad style) + nodeCollatRatioHelper := bigFloat(0) + + for _, r := range stakeInfoPerNode { subs, ok := subMap[hex.EncodeToString(r.Address)] if !ok { continue } - sub := subs[0] + sub := subs[0] // RPL min/max collateral notifications are always unique per user var alertConditionMet bool = false - if sub.EventThreshold >= 0 { - var threshold float64 = sub.EventThreshold - if threshold == 0 { - threshold = 1.0 - } - if eventName == types.RocketpoolCollateralMaxReached { - alertConditionMet = r.RPLStake.bigFloat().Cmp(r.RPLStakeMax.bigFloat().Mul(r.RPLStakeMax.bigFloat(), bigFloat(threshold))) >= 1 + // according to app logic, sub.EventThreshold can be +- [0.9 to 1.5] for CollateralMax after manually changed by the user + // this corresponds to a collateral range of 140% to 200% currently shown in the app UI; so +- 0.5 allows us to compare to the actual collat ratio + // for CollateralMin it can be 1.0 to 4.0 if manually changed, to represent 10% to 40% + // 0 in both cases if not modified + var threshold float64 = sub.EventThreshold + if threshold == 0 { + threshold = 1.0 // default case + } + inverse := false + if eventName == types.RocketpoolCollateralMaxReached { + if threshold < 0 { + threshold *= -1 } else { - alertConditionMet = r.RPLStake.bigFloat().Cmp(r.RPLStakeMin.bigFloat().Mul(r.RPLStakeMin.bigFloat(), bigFloat(threshold))) <= -1 + inverse = true } + threshold += 0.5 + + // 100% (of bonded eth) + nodeCollatRatioHelper.Quo(r.RPLStakeMax.bigFloat(), maxRPLCollatRatio) } else { - if eventName == types.RocketpoolCollateralMaxReached { - alertConditionMet = r.RPLStake.bigFloat().Cmp(r.RPLStakeMax.bigFloat().Mul(r.RPLStakeMax.bigFloat(), bigFloat(sub.EventThreshold*-1))) <= -1 - } else { - alertConditionMet = r.RPLStake.bigFloat().Cmp(r.RPLStakeMax.bigFloat().Mul(r.RPLStakeMin.bigFloat(), bigFloat(sub.EventThreshold*-1))) >= -1 - } + threshold /= 10.0 + + // 100% (of borrowed eth) + nodeCollatRatioHelper.Quo(r.RPLStakeMin.bigFloat(), minRPLCollatRatio) + } + + nodeCollatRatio, _ := nodeCollatRatioHelper.Quo(r.RPLStake.bigFloat(), nodeCollatRatioHelper).Float64() + + alertConditionMet = nodeCollatRatio <= threshold + if inverse { + // handle special case for max collateral: notify if *above* selected amount + alertConditionMet = !alertConditionMet } if !alertConditionMet { @@ -3017,7 +3052,7 @@ func collectRocketpoolRPLCollateralNotifications(notificationsByUserID map[uint6 if sub.LastEpoch != nil { lastSentEpoch := *sub.LastEpoch - if lastSentEpoch >= epoch-80 || epoch < sub.CreatedEpoch { + if lastSentEpoch >= epoch-225 || epoch < sub.CreatedEpoch { continue } } @@ -3028,6 +3063,7 @@ func collectRocketpoolRPLCollateralNotifications(notificationsByUserID map[uint6 Epoch: epoch, EventFilter: sub.EventFilter, EventName: eventName, + ExtraData: strings.TrimRight(strings.TrimRight(fmt.Sprintf("%.2f", threshold*100), "0"), "."), UnsubscribeHash: sub.UnsubscribeHash, } if _, exists := notificationsByUserID[*sub.UserID]; !exists { diff --git a/services/rewards.go b/services/rewards.go index caa79fe69b..ba4f153965 100644 --- a/services/rewards.go +++ b/services/rewards.go @@ -28,9 +28,9 @@ func GetValidatorHist(validatorArr []uint64, currency string, start uint64, end var pricesDb []types.Price // we get prices with a 1 day buffer to so we have no problems in different time zones - var oneDay = 24 * 60 * 60 + var oneDay = uint64(24 * 60 * 60) err = db.WriterDb.Select(&pricesDb, - `select ts, eur, usd, gbp, cad, jpy, cny, rub, aud from price where ts >= TO_TIMESTAMP($1) and ts <= TO_TIMESTAMP($2) order by ts desc`, start-uint64(oneDay), end+uint64(oneDay)) + `select ts, eur, usd, gbp, cad, jpy, cny, rub, aud from price where ts >= TO_TIMESTAMP($1) and ts <= TO_TIMESTAMP($2) order by ts desc`, start-oneDay, end+oneDay) if err != nil { logger.Errorf("error getting prices: %v", err) } diff --git a/services/services.go b/services/services.go index c11a3fde3e..2269101f38 100644 --- a/services/services.go +++ b/services/services.go @@ -377,14 +377,14 @@ func epochUpdater(wg *sync.WaitGroup) { } } - // latest exportered finalized epoch - var latestFinalized uint64 - err = db.WriterDb.Get(&latestFinalized, "SELECT COALESCE(MAX(epoch), 0) FROM epochs where finalized is true") + // latest exported finalized epoch + + latestFinalizedEpoch, err := db.GetLatestFinalizedEpoch() if err != nil { logger.Errorf("error retrieving latest exported finalized epoch from the database: %v", err) } else { cacheKey := fmt.Sprintf("%d:frontend:latestFinalized", utils.Config.Chain.ClConfig.DepositChainID) - err := cache.TieredCache.SetUint64(cacheKey, latestFinalized, time.Hour*24) + err := cache.TieredCache.SetUint64(cacheKey, latestFinalizedEpoch, time.Hour*24) if err != nil { logger.Errorf("error caching latestFinalizedEpoch: %v", err) } @@ -691,6 +691,18 @@ func getIndexPageData() (*types.IndexPageData, error) { return nil, fmt.Errorf("error retrieving eth1 deposits: %v", err) } + if deposit.Total == 0 { // see if there are any genesis validators + err = db.ReaderDb.Get(&deposit.Total, "SELECT COALESCE(MAX(validatorindex), 0) FROM validators") + if err != nil { + return nil, fmt.Errorf("error retrieving max validator index: %v", err) + } + + if deposit.Total > 0 { + deposit.Total = (deposit.Total + 1) * 32 + deposit.BlockTs = time.Now() + } + } + threshold, err := db.GetDepositThresholdTime() if err != nil { logger.WithError(err).Error("error could not calculate threshold time") @@ -785,8 +797,9 @@ func getIndexPageData() (*types.IndexPageData, error) { } data.ScheduledCount = scheduledCount + latestFinalizedEpoch := LatestFinalizedEpoch() var epochs []*types.IndexPageDataEpochs - err = db.ReaderDb.Select(&epochs, `SELECT epoch, finalized , eligibleether, globalparticipationrate, votedether FROM epochs ORDER BY epochs DESC LIMIT 15`) + err = db.ReaderDb.Select(&epochs, `SELECT epoch, (epoch <= $1) AS finalized , eligibleether, globalparticipationrate, votedether FROM epochs ORDER BY epochs DESC LIMIT 15`, latestFinalizedEpoch) if err != nil { return nil, fmt.Errorf("error retrieving index epoch data: %v", err) } @@ -905,7 +918,7 @@ func getIndexPageData() (*types.IndexPageData, error) { epochLowerBound = epoch - 1600 } var epochHistory []*types.IndexPageEpochHistory - err = db.WriterDb.Select(&epochHistory, "SELECT epoch, eligibleether, validatorscount, finalized, averagevalidatorbalance FROM epochs WHERE epoch < $1 and epoch > $2 ORDER BY epoch", epoch, epochLowerBound) + err = db.WriterDb.Select(&epochHistory, "SELECT epoch, eligibleether, validatorscount, (epoch <= $3) AS finalized, averagevalidatorbalance FROM epochs WHERE epoch < $1 and epoch > $2 ORDER BY epoch", epoch, epochLowerBound, latestFinalizedEpoch) if err != nil { return nil, fmt.Errorf("error retrieving staked ether history: %v", err) } @@ -1319,7 +1332,7 @@ func getGasNowData() (*types.GasNowPageData, error) { err = client.Call(&raw, "txpool_content") if err != nil { - utils.LogFatal(err, "error getting raw json data from txpool_content", 0) + return nil, fmt.Errorf("error getting raw json data from txpool_content: %w", err) } txPoolContent := &TxPoolContent{} @@ -1481,24 +1494,22 @@ func mempoolUpdater(wg *sync.WaitGroup) { func burnUpdater(wg *sync.WaitGroup) { firstRun := true - for { + for ; ; time.Sleep(time.Minute * 15) { // only update once every 15 minutes data, err := getBurnPageData() if err != nil { logger.Errorf("error retrieving burn page data: %v", err) - time.Sleep(time.Second * 30) continue } cacheKey := fmt.Sprintf("%d:frontend:burn", utils.Config.Chain.ClConfig.DepositChainID) err = cache.TieredCache.Set(cacheKey, data, time.Hour*24) if err != nil { - logger.Errorf("error caching relaysData: %v", err) + logger.Errorf("error caching burn data: %v", err) } if firstRun { logger.Infof("initialized burn updater") wg.Done() firstRun = false } - time.Sleep(time.Minute) } } @@ -1509,16 +1520,23 @@ func getBurnPageData() (*types.BurnPageData, error) { latestEpoch := LatestEpoch() latestBlock := LatestEth1BlockNumber() + // Check db to have at least one entry (will error otherwise anyway) + burnedFeesCount := 0 + if err := db.ReaderDb.Get(&burnedFeesCount, "SELECT COUNT(*) FROM chart_series WHERE indicator = 'BURNED_FEES'"); err != nil { + return nil, fmt.Errorf("error get BURNED_FEES count from chart_series: %w", err) + } + if burnedFeesCount <= 0 { + return data, nil + } + // Retrieve the total amount of burned Ether - err := db.ReaderDb.Get(&data.TotalBurned, "select sum(value) from chart_series where indicator = 'BURNED_FEES';") - if err != nil { - return nil, fmt.Errorf("error retrieving total burned amount from chart_series table: %v", err) + if err := db.ReaderDb.Get(&data.TotalBurned, "SELECT SUM(value) FROM chart_series WHERE indicator = 'BURNED_FEES'"); err != nil { + return nil, fmt.Errorf("error retrieving total burned amount from chart_series table: %w", err) } cutOff := time.Time{} - err = db.ReaderDb.Get(&cutOff, "select (select max(time) from chart_series where indicator = 'BURNED_FEES') + interval '24 hours';") - if err != nil { - return nil, fmt.Errorf("error retrieving cutoff date from chart_series table: %v", err) + if err := db.ReaderDb.Get(&cutOff, "SELECT ( SELECT MAX(time) FROM chart_series WHERE indicator = 'BURNED_FEES' ) + interval '24 hours'"); err != nil { + return nil, fmt.Errorf("error retrieving cutoff date from chart_series table: %w", err) } cutOffEpoch := utils.TimeToEpoch(cutOff) @@ -1530,14 +1548,14 @@ func getBurnPageData() (*types.BurnPageData, error) { // db.ReaderDb.Get(&blockLastDay) additionalBurned := float64(0) - err = db.ReaderDb.Get(&additionalBurned, "select COALESCE(SUM(exec_base_fee_per_gas::numeric * exec_gas_used::numeric), 0) as burnedfees from blocks where epoch > $1", cutOffEpoch) + err := db.ReaderDb.Get(&additionalBurned, "SELECT COALESCE(SUM(exec_base_fee_per_gas::numeric * exec_gas_used::numeric), 0) AS burnedfees FROM blocks WHERE epoch > $1", cutOffEpoch) if err != nil { return nil, fmt.Errorf("error retrieving additional burned eth from blocks table: %v", err) } // logger.Infof("additonal burn: %v", additionalBurned) data.TotalBurned += additionalBurned - err = db.ReaderDb.Get(&data.BurnRate1h, "select COALESCE(SUM(exec_base_fee_per_gas::numeric * exec_gas_used::numeric) / 60, 0) as burnedfees from blocks where epoch > $1", latestEpoch-10) + err = db.ReaderDb.Get(&data.BurnRate1h, "SELECT COALESCE(SUM(exec_base_fee_per_gas::numeric * exec_gas_used::numeric) / 60, 0) AS burnedfees FROM blocks WHERE epoch > $1", latestEpoch-10) if err != nil { return nil, fmt.Errorf("error retrieving burn rate (1h) from blocks table: %v", err) } @@ -1548,7 +1566,13 @@ func getBurnPageData() (*types.BurnPageData, error) { // } // swap this for GetEpochIncomeHistory in the future - income, err := db.BigtableClient.GetValidatorIncomeDetailsHistory([]uint64{}, latestEpoch-10, latestBlock) + + validators, err := db.GetValidatorIndices() + if err != nil { + return nil, err + } + + income, err := db.BigtableClient.GetValidatorIncomeDetailsHistory(validators, latestEpoch-10, latestEpoch) if err != nil { logger.WithError(err).Error("error getting validator income history") } @@ -1667,6 +1691,7 @@ func getBurnPageData() (*types.BurnPageData, error) { func latestExportedStatisticDayUpdater(wg *sync.WaitGroup) { firstRun := true + cacheKey := fmt.Sprintf("%d:frontend:lastExportedStatisticDay", utils.Config.Chain.ClConfig.DepositChainID) for { lastDay, err := db.GetLastExportedStatisticDay() if err != nil { @@ -1675,7 +1700,6 @@ func latestExportedStatisticDayUpdater(wg *sync.WaitGroup) { continue } - cacheKey := fmt.Sprintf("%d:frontend:lastExportedStatisticDay", utils.Config.Chain.ClConfig.DepositChainID) err = cache.TieredCache.Set(cacheKey, lastDay, time.Hour*24) if err != nil { logger.Errorf("error caching last exported statistics day: %v", err) @@ -1691,14 +1715,16 @@ func latestExportedStatisticDayUpdater(wg *sync.WaitGroup) { } // LatestExportedStatisticDay will return the last exported day in the validator_stats table -func LatestExportedStatisticDay() uint64 { +func LatestExportedStatisticDay() (uint64, error) { cacheKey := fmt.Sprintf("%d:frontend:lastExportedStatisticDay", utils.Config.Chain.ClConfig.DepositChainID) if wanted, err := cache.TieredCache.GetUint64WithLocalTimeout(cacheKey, time.Second*5); err == nil { - return wanted - } else { - logger.Errorf("error retrieving last exported statistics day from cache: %v", err) + return wanted, nil + } + wanted, err := db.GetLastExportedStatisticDay() + + if err != nil { + return 0, err } - wanted, _ := db.GetLastExportedStatisticDay() - return wanted + return wanted, nil } diff --git a/services/status.go b/services/status.go index cd9262a4c1..ab7cd969bf 100644 --- a/services/status.go +++ b/services/status.go @@ -30,6 +30,6 @@ func ReportStatus(name, status string, metadata *json.RawMessage) { `, name, execName, version, pid, status, metadata) if err != nil { - logger.Errorf("error reporting service status: %v", err) + utils.LogError(err, "error reporting service status", 0, map[string]interface{}{"name": name, "status": status}) } } diff --git a/static/js/dashboard.js b/static/js/dashboard.js index ea20c2e2b0..c9e681da46 100644 --- a/static/js/dashboard.js +++ b/static/js/dashboard.js @@ -692,6 +692,24 @@ $(document).ready(function () { } create_validators_typeahead("input[aria-controls='validators']", "#validators") + var timeWait = 0 + var debounce = function (context, func) { + var timeout, result + + return function () { + var args = arguments, + later = function () { + timeout = null + result = func.apply(context, args) + } + clearTimeout(timeout) + timeout = setTimeout(later, timeWait) + if (!timeout) { + result = func.apply(context, args) + } + return result + } + } var bhValidators = new Bloodhound({ datumTokenizer: Bloodhound.tokenizers.whitespace, queryTokenizer: Bloodhound.tokenizers.whitespace, @@ -700,9 +718,19 @@ $(document).ready(function () { }, remote: { url: "/search/indexed_validators/%QUERY", - wildcard: "%QUERY", + // use prepare hook to modify the rateLimitWait parameter on input changes + // NOTE: we only need to do this for the first function because testing showed that queries are executed/queued in order + // No need to update `timeWait` multiple times. + prepare: function (_, settings) { + var cur_query = $(".typeahead-dashboard").val() + timeWait = 4000 - Math.min(cur_query.length, 5) * 500 + // "wildcard" can't be used anymore, need to set query wildcard ourselves now + settings.url = settings.url.replace("%QUERY", encodeURIComponent(cur_query)) + return settings + }, }, }) + bhValidators.remote.transport._get = debounce(bhValidators.remote.transport, bhValidators.remote.transport._get) var bhPubkey = new Bloodhound({ datumTokenizer: Bloodhound.tokenizers.whitespace, queryTokenizer: Bloodhound.tokenizers.whitespace, @@ -714,6 +742,7 @@ $(document).ready(function () { wildcard: "%QUERY", }, }) + bhPubkey.remote.transport._get = debounce(bhPubkey.remote.transport, bhPubkey.remote.transport._get) var bhEth1Addresses = new Bloodhound({ datumTokenizer: Bloodhound.tokenizers.whitespace, queryTokenizer: Bloodhound.tokenizers.whitespace, @@ -725,6 +754,7 @@ $(document).ready(function () { wildcard: "%QUERY", }, }) + bhEth1Addresses.remote.transport._get = debounce(bhEth1Addresses.remote.transport, bhEth1Addresses.remote.transport._get) var bhName = new Bloodhound({ datumTokenizer: Bloodhound.tokenizers.whitespace, queryTokenizer: Bloodhound.tokenizers.whitespace, @@ -736,6 +766,7 @@ $(document).ready(function () { wildcard: "%QUERY", }, }) + bhName.remote.transport._get = debounce(bhName.remote.transport, bhName.remote.transport._get) var bhGraffiti = new Bloodhound({ datumTokenizer: Bloodhound.tokenizers.whitespace, queryTokenizer: Bloodhound.tokenizers.whitespace, @@ -747,6 +778,7 @@ $(document).ready(function () { wildcard: "%QUERY", }, }) + bhGraffiti.remote.transport._get = debounce(bhGraffiti.remote.transport, bhGraffiti.remote.transport._get) $(".typeahead-dashboard").typeahead( { @@ -1390,6 +1422,9 @@ function createIncomeChart(income, executionIncomeHistory) { }, pointInterval: 24 * 3600 * 1000, }, + series: { + turboThreshold: 10000, + }, }, xAxis: { type: "datetime", diff --git a/static/js/datatable_loader.js b/static/js/datatable_loader.js new file mode 100644 index 0000000000..fc28f8e1c5 --- /dev/null +++ b/static/js/datatable_loader.js @@ -0,0 +1,56 @@ +/** + * function dataTableLoader(path) + * used to create an ajax function for a DataTable. + * This function is used to load data from the server for a DataTable. + * It is debounced to avoid multiple requests to the server when the user clicks on the pagination buttons. + * There is also a retry mechanism that retries the server request if the request fails. + * @param path: Path to load the table data from the server + */ +function dataTableLoader(path) { + const MAX_RETRIES = 5 + const DEBOUNCE_DELAY = 500 + const RETRY_DELAY = 1000 + + let retries = 0 + let timeoutId + + const debounce = (func, delay) => { + let debounceTimer + return function () { + const context = this + const args = arguments + clearTimeout(debounceTimer) + debounceTimer = setTimeout(() => func.apply(context, args), delay) + } + } + + const doFetch = (tableData, callback) => { + fetch(`${path}?${$.param(tableData)}`) + .then((response) => { + if (!response.ok) { + throw new Error(`Failed with status: ${response.status}`) + } + return response.json() + }) + .then((data) => { + callback(data) + }) + .catch((err) => { + if (retries < MAX_RETRIES) { + retries++ + timeoutId = setTimeout(() => doFetch(tableData, callback), RETRY_DELAY * (retries + 1)) + } else { + console.error("Failed to fetch data for path: ", path, "with error: ", err) + } + }) + } + + const fetchWithRetry = (tableData, callback) => { + clearTimeout(timeoutId) // Clear any pending retries. + retries = 0 // Reset retry count. + doFetch(tableData, callback) + } + const debouncedFetchData = debounce(fetchWithRetry, DEBOUNCE_DELAY) + + return debouncedFetchData +} diff --git a/static/js/layout.js b/static/js/layout.js index 36aa886e8b..8390ecf1cc 100644 --- a/static/js/layout.js +++ b/static/js/layout.js @@ -223,7 +223,30 @@ $(document).ready(function () { // set maxParallelRequests to number of datasets queried in each search // make sure this is set in every one bloodhound object - let requestNum = 9 + let requestNum = 10 + var timeWait = 0 + + // used to overwrite Bloodhounds "transport._get" function which handles the rateLimitWait parameter + // since we can't access the closure variable anymore (?) + // copied and adapted from typeahead.js source code (https://github.com/twitter/typeahead.js/blob/master/dist/typeahead.bundle.js#L106) + // -> so we have control via `timeWait` now + var debounce = function (context, func) { + var timeout, result + + return function () { + var args = arguments, + later = function () { + timeout = null + result = func.apply(context, args) + } + clearTimeout(timeout) + timeout = setTimeout(later, timeWait) + if (!timeout) { + result = func.apply(context, args) + } + return result + } + } var bhValidators = new Bloodhound({ datumTokenizer: Bloodhound.tokenizers.whitespace, @@ -233,10 +256,20 @@ $(document).ready(function () { }, remote: { url: "/search/validators/%QUERY", - wildcard: "%QUERY", + // use prepare hook to modify the rateLimitWait parameter on input changes + // NOTE: we only need to do this for the first function because testing showed that queries are executed/queued in order + // No need to update `timeWait` multiple times. + prepare: function (_, settings) { + var cur_query = $("input.typeahead.tt-input").val() + timeWait = 4000 - Math.min(cur_query.length, 5) * 500 + // "wildcard" can't be used anymore, need to set query wildcard ourselves now + settings.url = settings.url.replace("%QUERY", encodeURIComponent(cur_query)) + return settings + }, maxPendingRequests: requestNum, }, }) + bhValidators.remote.transport._get = debounce(bhValidators.remote.transport, bhValidators.remote.transport._get) var bhEns = new Bloodhound({ datumTokenizer: Bloodhound.tokenizers.whitespace, @@ -253,6 +286,7 @@ $(document).ready(function () { }, }, }) + bhEns.remote.transport._get = debounce(bhEns.remote.transport, bhEns.remote.transport._get) var bhSlots = new Bloodhound({ datumTokenizer: Bloodhound.tokenizers.whitespace, @@ -266,6 +300,7 @@ $(document).ready(function () { maxPendingRequests: requestNum, }, }) + bhSlots.remote.transport._get = debounce(bhSlots.remote.transport, bhSlots.remote.transport._get) var bhBlocks = new Bloodhound({ datumTokenizer: Bloodhound.tokenizers.whitespace, @@ -279,6 +314,7 @@ $(document).ready(function () { maxPendingRequests: requestNum, }, }) + bhBlocks.remote.transport._get = debounce(bhBlocks.remote.transport, bhBlocks.remote.transport._get) var bhTransactions = new Bloodhound({ datumTokenizer: Bloodhound.tokenizers.whitespace, @@ -292,6 +328,7 @@ $(document).ready(function () { maxPendingRequests: requestNum, }, }) + bhTransactions.remote.transport._get = debounce(bhTransactions.remote.transport, bhTransactions.remote.transport._get) var bhGraffiti = new Bloodhound({ datumTokenizer: Bloodhound.tokenizers.whitespace, @@ -305,6 +342,7 @@ $(document).ready(function () { maxPendingRequests: requestNum, }, }) + bhGraffiti.remote.transport._get = debounce(bhGraffiti.remote.transport, bhGraffiti.remote.transport._get) var bhEpochs = new Bloodhound({ datumTokenizer: Bloodhound.tokenizers.whitespace, @@ -318,6 +356,7 @@ $(document).ready(function () { maxPendingRequests: requestNum, }, }) + bhEpochs.remote.transport._get = debounce(bhEpochs.remote.transport, bhEpochs.remote.transport._get) var bhEth1Accounts = new Bloodhound({ datumTokenizer: Bloodhound.tokenizers.whitespace, @@ -331,6 +370,7 @@ $(document).ready(function () { maxPendingRequests: requestNum, }, }) + bhEth1Accounts.remote.transport._get = debounce(bhEth1Accounts.remote.transport, bhEth1Accounts.remote.transport._get) var bhValidatorsByAddress = new Bloodhound({ datumTokenizer: Bloodhound.tokenizers.whitespace, @@ -344,6 +384,21 @@ $(document).ready(function () { maxPendingRequests: requestNum, }, }) + bhValidatorsByAddress.remote.transport._get = debounce(bhValidatorsByAddress.remote.transport, bhValidatorsByAddress.remote.transport._get) + + var bhPubkey = new Bloodhound({ + datumTokenizer: Bloodhound.tokenizers.whitespace, + queryTokenizer: Bloodhound.tokenizers.whitespace, + identify: function (obj) { + return obj.index + }, + remote: { + url: "/search/validators_by_pubkey/%QUERY", + wildcard: "%QUERY", + maxPendingRequests: requestNum, + }, + }) + bhPubkey.remote.transport._get = debounce(bhPubkey.remote.transport, bhPubkey.remote.transport._get) // before adding datasets make sure requestNum is set to the correct value $(".typeahead").typeahead( @@ -365,6 +420,18 @@ $(document).ready(function () { }, }, }, + { + limit: 5, + name: "pubkeys", + source: bhPubkey, + display: "pubkey", + templates: { + header: '

Validators by Public Key

', + suggestion: function (data) { + return `
${data.pubkey}
` + }, + }, + }, { limit: 5, name: "ens", diff --git a/static/js/notificationsCenter.js b/static/js/notificationsCenter.js index c9016e0280..127bc22cd3 100755 --- a/static/js/notificationsCenter.js +++ b/static/js/notificationsCenter.js @@ -5,6 +5,25 @@ const VALIDATOR_EVENTS = ["validator_attestation_missed", "validator_proposal_mi // const MONITORING_EVENTS = ['monitoring_machine_offline', 'monitoring_hdd_almostfull', 'monitoring_cpu_load'] function create_typeahead(input_container) { + var timeWait = 0 + var debounce = function (context, func) { + var timeout, result + + return function () { + var args = arguments, + later = function () { + timeout = null + result = func.apply(context, args) + } + clearTimeout(timeout) + timeout = setTimeout(later, timeWait) + if (!timeout) { + result = func.apply(context, args) + } + return result + } + } + var bhValidators = new Bloodhound({ datumTokenizer: Bloodhound.tokenizers.whitespace, queryTokenizer: Bloodhound.tokenizers.whitespace, @@ -13,9 +32,19 @@ function create_typeahead(input_container) { }, remote: { url: "/search/validators/%QUERY", - wildcard: "%QUERY", + // use prepare hook to modify the rateLimitWait parameter on input changes + // NOTE: we only need to do this for the first function because testing showed that queries are executed/queued in order + // No need to update `timeWait` multiple times. + prepare: function (_, settings) { + var cur_query = $(input_container).val() + timeWait = 4000 - Math.min(cur_query.length, 5) * 500 + // "wildcard" can't be used anymore, need to set query wildcard ourselves now + settings.url = settings.url.replace("%QUERY", encodeURIComponent(cur_query)) + return settings + }, }, }) + bhValidators.remote.transport._get = debounce(bhValidators.remote.transport, bhValidators.remote.transport._get) var bhName = new Bloodhound({ datumTokenizer: Bloodhound.tokenizers.whitespace, queryTokenizer: Bloodhound.tokenizers.whitespace, @@ -27,6 +56,7 @@ function create_typeahead(input_container) { wildcard: "%QUERY", }, }) + bhName.remote.transport._get = debounce(bhName.remote.transport, bhName.remote.transport._get) var bhGraffiti = new Bloodhound({ datumTokenizer: Bloodhound.tokenizers.whitespace, queryTokenizer: Bloodhound.tokenizers.whitespace, @@ -38,6 +68,7 @@ function create_typeahead(input_container) { wildcard: "%QUERY", }, }) + bhGraffiti.remote.transport._get = debounce(bhGraffiti.remote.transport, bhGraffiti.remote.transport._get) var bhEth1Addresses = new Bloodhound({ datumTokenizer: Bloodhound.tokenizers.whitespace, queryTokenizer: Bloodhound.tokenizers.whitespace, @@ -49,6 +80,7 @@ function create_typeahead(input_container) { wildcard: "%QUERY", }, }) + bhEth1Addresses.remote.transport._get = debounce(bhEth1Addresses.remote.transport, bhEth1Addresses.remote.transport._get) $(input_container).typeahead( { minLength: 1, diff --git a/static/js/validatorRewards.js b/static/js/validatorRewards.js index 2c14400933..16d8bcd5a9 100644 --- a/static/js/validatorRewards.js +++ b/static/js/validatorRewards.js @@ -7,6 +7,25 @@ var subsTable = null // let validators = [] function create_typeahead(input_container) { + var timeWait = 0 + var debounce = function (context, func) { + var timeout, result + + return function () { + var args = arguments, + later = function () { + timeout = null + result = func.apply(context, args) + } + clearTimeout(timeout) + timeout = setTimeout(later, timeWait) + if (!timeout) { + result = func.apply(context, args) + } + return result + } + } + var bhValidators = new Bloodhound({ datumTokenizer: Bloodhound.tokenizers.whitespace, queryTokenizer: Bloodhound.tokenizers.whitespace, @@ -15,9 +34,19 @@ function create_typeahead(input_container) { }, remote: { url: "/search/indexed_validators/%QUERY", - wildcard: "%QUERY", + // use prepare hook to modify the rateLimitWait parameter on input changes + // NOTE: we only need to do this for the first function because testing showed that queries are executed/queued in order + // No need to update `timeWait` multiple times. + prepare: function (_, settings) { + var cur_query = $(input_container).val() + timeWait = 4000 - Math.min(cur_query.length, 5) * 500 + // "wildcard" can't be used anymore, need to set query wildcard ourselves now + settings.url = settings.url.replace("%QUERY", encodeURIComponent(cur_query)) + return settings + }, }, }) + bhValidators.remote.transport._get = debounce(bhValidators.remote.transport, bhValidators.remote.transport._get) var bhEth1Addresses = new Bloodhound({ datumTokenizer: Bloodhound.tokenizers.whitespace, queryTokenizer: Bloodhound.tokenizers.whitespace, @@ -29,6 +58,7 @@ function create_typeahead(input_container) { wildcard: "%QUERY", }, }) + bhEth1Addresses.remote.transport._get = debounce(bhEth1Addresses.remote.transport, bhEth1Addresses.remote.transport._get) var bhName = new Bloodhound({ datumTokenizer: Bloodhound.tokenizers.whitespace, queryTokenizer: Bloodhound.tokenizers.whitespace, @@ -40,6 +70,7 @@ function create_typeahead(input_container) { wildcard: "%QUERY", }, }) + bhName.remote.transport._get = debounce(bhName.remote.transport, bhName.remote.transport._get) var bhGraffiti = new Bloodhound({ datumTokenizer: Bloodhound.tokenizers.whitespace, queryTokenizer: Bloodhound.tokenizers.whitespace, @@ -51,6 +82,7 @@ function create_typeahead(input_container) { wildcard: "%QUERY", }, }) + bhGraffiti.remote.transport._get = debounce(bhGraffiti.remote.transport, bhGraffiti.remote.transport._get) $(input_container).typeahead( { diff --git a/templates/correlations.html b/templates/correlations.html index ae0b143aa0..28233ac079 100644 --- a/templates/correlations.html +++ b/templates/correlations.html @@ -68,7 +68,7 @@ text: "Correlation between " + x + " and " + y, }, subtitle: { - text: document.ontouchstart === undefined ? 'Source: etherchain.org
Click and drag in the plot area to zoom in' : 'Source: etherchain.org
Pinch the chart to zoom in', + text: document.ontouchstart === undefined ? 'Source: beaconcha.in
Click and drag in the plot area to zoom in' : 'Source: beaconcha.in
Pinch the chart to zoom in', }, xAxis: { title: { diff --git a/templates/deposits.html b/templates/deposits.html index 2181df2704..8430ad909b 100644 --- a/templates/deposits.html +++ b/templates/deposits.html @@ -177,13 +177,17 @@

Top Depo

Runner Up

- {{ if len .Stats.TopDepositors }}{{ (index .Stats.TopDepositors 1).DepositCount }}{{ else }}0{{ end }} Deposits + {{ if gt (len .Stats.TopDepositors) 1 }}{{ (index .Stats.TopDepositors 1).DepositCount }}{{ else }}0{{ end }} Deposits
- {{ if len .Stats.TopDepositors }}0x{{ (index .Stats.TopDepositors 1).Address }}{{ end }} + {{ if gt (len .Stats.TopDepositors) 1 }} + 0x{{ (index .Stats.TopDepositors 1).Address }} + {{ else }} + N/A + {{ end }}
diff --git a/templates/educationServices.html b/templates/educationServices.html deleted file mode 100644 index 7ca40b6425..0000000000 --- a/templates/educationServices.html +++ /dev/null @@ -1,547 +0,0 @@ -{{ define "js" }} -{{ end }} - -{{ define "css" }} - -{{ end }} - -{{ define "content" }} -
-
-
-
-

When One Learns, We All Learn

-

Community-Contributed Educational Materials

-
-
- - - - - education - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
-
-
-
-
-
-
-
-

Educational Material

-
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
NameAuthorTypeAssociationSupport
EthStaker CommunityEthStakerCommunityr/EthStakerGitcoin
Eth2 Study MasterSuperphiz, UnveticaCourser/EthStakerGitcoin
Eth2 newsBenjamin EdingtonNewsTeku, ConsenSysGitcoin
Attestant PostsJim McDonaldGeneral KnowledgeAttestant N/A
Beaconcha.in KBButtaGeneral Knowledgebeaconcha.in Gitcoin
Guide to Staking on EthereumSomer EsatGuideIndependentGitcoin
CoinCashew's Ethereum GuideCoincCashewGuideIndependentGitcoin
Rewards on Ethereum 2.0CodefiRewardsConsensysGitcoin
Eth2 Annotated SpecBenjamin EdingtonEth2 SpecTeku, ConsenSysGitcoin
Vitalik’s Annotated Ethereum 2.0 SpecVitalik ButerinEth2 SpecEthereum FoundationN/A
Serenity HandbookHsiao-Wei WangEth2 SpecEthereum FoundationN/A
Phase0 for HumansDanny Ryan, ProtoEth2 SpecEthereum FoundationN/A
Proto's Eth2 docsProtoEth2 SpecEthereum FoundationN/A
Serenity Design RationaleVitalik ButerinEth2 SpecEthereum FoundationN/A
Ethereum Sharding Research CompendiumVitalik ButerinEth2 SpecEthereum FoundationN/A
Beacon Digest Data ReportsRobust Incentives Group - Ethereum FoundationData ReportsEthereum FoundationN/A
-
-
-
-
-
-
-
-
-
-

Request to add your Education Service

-

- If you have an education service and want it to be listed above please create a pull request -

-
-
-
-
-
-
-{{ end }} diff --git a/templates/faq.html b/templates/faq.html deleted file mode 100644 index bbee6ed2be..0000000000 --- a/templates/faq.html +++ /dev/null @@ -1,28 +0,0 @@ -{{ define "js" }} -{{ end }} - -{{ define "css" }} -{{ end }} - -{{ define "content" }} - {{ with .Data }} -
-
-
-

FAQ

- -
-
-
-
-
Not available right now :(
-
-
-
- {{ end }} -{{ end }} diff --git a/templates/gasnow.html b/templates/gasnow.html index 1530615a4a..3ada10d750 100644 --- a/templates/gasnow.html +++ b/templates/gasnow.html @@ -84,8 +84,8 @@ } }], credits: { - href: "https://www.etherchain.org/tools/gasnow", - text: "etherchain.org" + href: "https://www.beaconcha.in/tools/gasnow", + text: "beaconcha.in" } }); @@ -124,7 +124,7 @@ { "name": "Transfer", "gas": 46109, - "url": "https://etherchain.org/tx/0x060babb87d37e81a2d97244f2866709e4ee19b27b93a5b7356be048cd193eaed" + "url": "https://beaconcha.in/tx/0x060babb87d37e81a2d97244f2866709e4ee19b27b93a5b7356be048cd193eaed" } ] }, @@ -138,7 +138,7 @@ { "name": "Transfer", "gas": 48481, - "url": "https://etherchain.org/tx/0x8f771b091c96d1ccb852c5df1ed4aff93755a4f4a5e922f000770c9646528ef6" + "url": "https://beaconcha.in/tx/0x8f771b091c96d1ccb852c5df1ed4aff93755a4f4a5e922f000770c9646528ef6" } ] }, @@ -152,7 +152,7 @@ { "name": "Transfer", "gas": 34718, - "url": "https://etherchain.org/tx/0x8f58ccac094237196f8fe2c0fb980faa80dd41d0c051ab692251e96a53cd19ef" + "url": "https://beaconcha.in/tx/0x8f58ccac094237196f8fe2c0fb980faa80dd41d0c051ab692251e96a53cd19ef" } ] }, @@ -166,17 +166,17 @@ { "name": "Swap", "gas": 129830, - "url": "https://etherchain.org/tx/0x5e1b0897800e005d02e9051aec269cedea73b2f59f7b208ae460b4027ebac575" + "url": "https://beaconcha.in/tx/0x5e1b0897800e005d02e9051aec269cedea73b2f59f7b208ae460b4027ebac575" }, { "name": "Add Liquidity", "gas": 445784, - "url": "https://etherchain.org/tx/0xc4d202633c8e27e41216b348f64f1f2c7f21cd6a79c2a457f0f920aefb616bbf" + "url": "https://beaconcha.in/tx/0xc4d202633c8e27e41216b348f64f1f2c7f21cd6a79c2a457f0f920aefb616bbf" }, { "name": "Remove Liquidity", "gas": 221722, - "url": "https://etherchain.org/tx/0xaa83ff4c83ad5ef62213b54c8774992661fb3864657edd85ed6991011af3cdbe" + "url": "https://beaconcha.in/tx/0xaa83ff4c83ad5ef62213b54c8774992661fb3864657edd85ed6991011af3cdbe" } ] }, @@ -190,17 +190,17 @@ { "name": "Swap", "gas": 105657, - "url": "https://etherchain.org/tx/0xdf5a48258d3b43899dcaa3e05754f0a70279fcfaf98746fad81fbb5bef8ce6c5" + "url": "https://beaconcha.in/tx/0xdf5a48258d3b43899dcaa3e05754f0a70279fcfaf98746fad81fbb5bef8ce6c5" }, { "name": "Add Liquidity", "gas": 131820, - "url": "https://etherchain.org/tx/0x814104644915370c9bcf23dd834a61839f3c4b6a225086b54fa3a3b0621bf249" + "url": "https://beaconcha.in/tx/0x814104644915370c9bcf23dd834a61839f3c4b6a225086b54fa3a3b0621bf249" }, { "name": "Remove Liquidity", "gas": 180244, - "url": "https://etherchain.org/tx/0xce0110cc934d1ed69efd7fabe4ba38ba557f57c614ed6137bc11d28b72f4bf39" + "url": "https://beaconcha.in/tx/0xce0110cc934d1ed69efd7fabe4ba38ba557f57c614ed6137bc11d28b72f4bf39" } ] }, @@ -214,7 +214,7 @@ { "name": "Swap", "gas": 104108, - "url": "https://etherchain.org/tx/0x3b5c243f02e8eded9b25a258b381429de83e1854d4eaafce456f79a04ffd4890" + "url": "https://beaconcha.in/tx/0x3b5c243f02e8eded9b25a258b381429de83e1854d4eaafce456f79a04ffd4890" } ] }, @@ -228,7 +228,7 @@ { "name": "Swap", "gas": 109253, - "url": "https://etherchain.org/tx/0xcf8ba925887ca185f8cd98fa986f728012b1f7b47f82d6a258ea9ac2aed8c811" + "url": "https://beaconcha.in/tx/0xcf8ba925887ca185f8cd98fa986f728012b1f7b47f82d6a258ea9ac2aed8c811" } ] }, @@ -242,7 +242,7 @@ { "name": "Swap", "gas": 114931, - "url": "https://etherchain.org/tx/0x0860f56786601da581dfe5f431583827136356b967b42482d37593fa00adfdc6" + "url": "https://beaconcha.in/tx/0x0860f56786601da581dfe5f431583827136356b967b42482d37593fa00adfdc6" } ] }, @@ -256,17 +256,17 @@ { "name": "Swap", "gas": 114651, - "url": "https://etherchain.org/tx/0x3b585927cf4e2c603fc4eef295c60ad6bcb446dfc9e44d8ec78072f6e7c52f14" + "url": "https://beaconcha.in/tx/0x3b585927cf4e2c603fc4eef295c60ad6bcb446dfc9e44d8ec78072f6e7c52f14" }, { "name": "AddLiquidity", "gas": 182725, - "url": "https://etherchain.org/tx/0x07ccba5e6f38c007c4795e1f8200b41931f18d7e4fa1dae2213493feaca5cc77" + "url": "https://beaconcha.in/tx/0x07ccba5e6f38c007c4795e1f8200b41931f18d7e4fa1dae2213493feaca5cc77" }, { "name": "RemoveLiquidityOneCoin", "gas": 162054, - "url": "https://etherchain.org/tx/0x4fefbfb2e260f2f0dc7d24d486e6f3ceedd17a90198dd45595dda8890cf878ac" + "url": "https://beaconcha.in/tx/0x4fefbfb2e260f2f0dc7d24d486e6f3ceedd17a90198dd45595dda8890cf878ac" } ] }, @@ -280,12 +280,12 @@ { "name": "Deposit L1->L2", "gas": 100712, - "url": "https://etherchain.org/tx/0x4c32d72ce839e67a360f2dba1d6b9bcb4607cc9652ddaf245c708534dcbe1608" + "url": "https://beaconcha.in/tx/0x4c32d72ce839e67a360f2dba1d6b9bcb4607cc9652ddaf245c708534dcbe1608" }, { "name": "Withdraw L2->L1", "gas": 267085, - "url": "https://etherchain.org/tx/0xbfaf3ab325112add5b0932a7c1b3387ccb6a498118963456c4c5958f9db82c73" + "url": "https://beaconcha.in/tx/0xbfaf3ab325112add5b0932a7c1b3387ccb6a498118963456c4c5958f9db82c73" } ] }, @@ -299,7 +299,7 @@ { "name": "Deposit L1->L2", "gas": 58952, - "url": "https://etherchain.org/tx/0xe16b21b5c8eb3ee61e15663502c7bec3a982e5ca2187ff98f7fe72e094b80386" + "url": "https://beaconcha.in/tx/0xe16b21b5c8eb3ee61e15663502c7bec3a982e5ca2187ff98f7fe72e094b80386" }, { "name": "Withdraw L2->L1", @@ -318,12 +318,12 @@ { "name": "Deposit L1->L2", "gas": 73298, - "url": "https://etherchain.org/tx/0xf7bcda493e68e63ea4be233022576ea08692a4ca48b010f280a61054012e498b" + "url": "https://beaconcha.in/tx/0xf7bcda493e68e63ea4be233022576ea08692a4ca48b010f280a61054012e498b" }, { "name": "Withdraw L2->L1", "gas": 85679, - "url": "https://etherchain.org/tx/0x221b0c69f52c036893819487a90b16e89f3681aa25947e0eb358319351f9835b" + "url": "https://beaconcha.in/tx/0x221b0c69f52c036893819487a90b16e89f3681aa25947e0eb358319351f9835b" } ] }, @@ -337,32 +337,32 @@ { "name": "Deposit USDC", "gas": 193404, - "url": "https://etherchain.org/tx/0x32f2e79436c3d542f0316043ecb91dc78a2791b1d6c4746b3986e53673c2269b" + "url": "https://beaconcha.in/tx/0x32f2e79436c3d542f0316043ecb91dc78a2791b1d6c4746b3986e53673c2269b" }, { "name": "Borrow USDC", "gas": 332745, - "url": "https://etherchain.org/tx/0x1691f02114226dfdac93e6dc28cd8ee93c21c834463679685830da767776cd71" + "url": "https://beaconcha.in/tx/0x1691f02114226dfdac93e6dc28cd8ee93c21c834463679685830da767776cd71" }, { "name": "Repay USDC", "gas": 177875, - "url": "https://etherchain.org/tx/0x730c8162759f10cc9f7a44bd5bc2779eb1d27f1eaf8ed3e47e1353462d656ec5" + "url": "https://beaconcha.in/tx/0x730c8162759f10cc9f7a44bd5bc2779eb1d27f1eaf8ed3e47e1353462d656ec5" }, { "name": "Withdraw USDC", "gas": 203573, - "url": "https://etherchain.org/tx/0x7aedc53924727ecfcf14b448777890c7d93966b646f396aa258eb8ce5b3a201b" + "url": "https://beaconcha.in/tx/0x7aedc53924727ecfcf14b448777890c7d93966b646f396aa258eb8ce5b3a201b" }, { "name": "Deposit ETH", "gas": 160537, - "url": "https://etherchain.org/tx/0xe3e82e90763cff73132ff68cb77361d0514cc2ad55a290a2ed13816395241071" + "url": "https://beaconcha.in/tx/0xe3e82e90763cff73132ff68cb77361d0514cc2ad55a290a2ed13816395241071" }, { "name": "Withdraw ETH", "gas": 138300, - "url": "https://etherchain.org/tx/0x550902ce5e3dd7fc07c637cee68a1c82cacbec61df90b3a8f71c7ac34e457437" + "url": "https://beaconcha.in/tx/0x550902ce5e3dd7fc07c637cee68a1c82cacbec61df90b3a8f71c7ac34e457437" } ] }, @@ -376,32 +376,32 @@ { "name": "Deposit USDC", "gas": 186695, - "url": "https://etherchain.org/tx/0xf0e49ed9f898b90f28528d6c3a5fd055e32e51c46c9cb28348fdfbae68751a4b" + "url": "https://beaconcha.in/tx/0xf0e49ed9f898b90f28528d6c3a5fd055e32e51c46c9cb28348fdfbae68751a4b" }, { "name": "Borrow USDC", "gas": 269145, - "url": "https://etherchain.org/tx/0xfdda54cf8e612ee39e4a65d4987df41524ddd88f22a934a8ddefa0302a8935d8" + "url": "https://beaconcha.in/tx/0xfdda54cf8e612ee39e4a65d4987df41524ddd88f22a934a8ddefa0302a8935d8" }, { "name": "Repay USDC", "gas": 210337, - "url": "https://etherchain.org/tx/0x9d424ad44891a092bd6ca00dc2f3de03e8a20b277b2dbd30add5c412448e4ff1" + "url": "https://beaconcha.in/tx/0x9d424ad44891a092bd6ca00dc2f3de03e8a20b277b2dbd30add5c412448e4ff1" }, { "name": "Withdraw USDC", "gas": 204977, - "url": "https://etherchain.org/tx/0x5c1818db2b9d992d2b79c729f0aed66329a6ee02fa62f32c03e3220e035a2501" + "url": "https://beaconcha.in/tx/0x5c1818db2b9d992d2b79c729f0aed66329a6ee02fa62f32c03e3220e035a2501" }, { "name": "Deposit ETH", "gas": 190595, - "url": "https://etherchain.org/tx/0xea4afe0faecada55fdf9f73e9aad7e91e8ba5af7a6d9c25a9c0ea5403b9ab870" + "url": "https://beaconcha.in/tx/0xea4afe0faecada55fdf9f73e9aad7e91e8ba5af7a6d9c25a9c0ea5403b9ab870" }, { "name": "Withdraw ETH", "gas": 251833, - "url": "https://etherchain.org/tx/0x67fec4b263c6c3755dd348d798784a209b7db06f1c92f5428282aab8fd965190" + "url": "https://beaconcha.in/tx/0x67fec4b263c6c3755dd348d798784a209b7db06f1c92f5428282aab8fd965190" } ] }, @@ -439,12 +439,12 @@ { "name": "Deposit", "gas": 952910, - "url": "https://etherchain.org/tx/0xecd116e2d7701801b64321d671ac1b82d0d58b35390dfc4c92e6142ea579d9ff" + "url": "https://beaconcha.in/tx/0xecd116e2d7701801b64321d671ac1b82d0d58b35390dfc4c92e6142ea579d9ff" }, { "name": "Withdraw", "gas": 366978, - "url": "https://etherchain.org/tx/0x86240e07d4bd38d015c79dad768551611b58554998dbc7cb63e2b3fbf7a50915" + "url": "https://beaconcha.in/tx/0x86240e07d4bd38d015c79dad768551611b58554998dbc7cb63e2b3fbf7a50915" } ] }, @@ -458,7 +458,7 @@ { "name": "Register Account", "gas": 389335, - "url": "https://etherchain.org/tx/0x3c52625c44e8435fcb43c33bc199de9ae95d7c8abfbc72f5f45b1fe5f2e8588c" + "url": "https://beaconcha.in/tx/0x3c52625c44e8435fcb43c33bc199de9ae95d7c8abfbc72f5f45b1fe5f2e8588c" }, { "name": "Post NFT", @@ -467,12 +467,12 @@ { "name": "Buy NFT", "gas": 189873, - "url": "https://etherchain.org/tx/0x4cbd1df48af864df822bab77759551e5ea7b2c8e635cfb4f231489a03225ac6e" + "url": "https://beaconcha.in/tx/0x4cbd1df48af864df822bab77759551e5ea7b2c8e635cfb4f231489a03225ac6e" }, { "name": "Cancel Order", "gas": 74468, - "url": "https://etherchain.org/tx/0x7a37d86bfe77aea4d6947f07e05177e9cf7ef0a53284cc737d87fb26fd4e0a07" + "url": "https://beaconcha.in/tx/0x7a37d86bfe77aea4d6947f07e05177e9cf7ef0a53284cc737d87fb26fd4e0a07" } ] } @@ -532,7 +532,7 @@ // console.log('gasnow data', response) this.page = response; - document.title = (response.data.rapid / 1e9).toFixed(0) + "-" + (response.data.fast / 1e9).toFixed(0) + " GWei | Ethereum (ETH) Mainnet - GasNow - etherchain.org - " + new Date().getFullYear(); + document.title = (response.data.rapid / 1e9).toFixed(0) + "-" + (response.data.fast / 1e9).toFixed(0) + " GWei | Ethereum (ETH) Mainnet - GasNow - beaconcha.in - " + new Date().getFullYear(); }.bind(this)); this.updateIn = 5; this.progress = 0; diff --git a/templates/layout.html b/templates/layout.html index afd6dd7082..dd4808d033 100644 --- a/templates/layout.html +++ b/templates/layout.html @@ -257,7 +257,7 @@
- By using our site you agree to our use of cookies to deliver a better user experience. + By using our site you agree to our use of cookies to deliver a better user experience.
@@ -281,10 +281,10 @@
Legal Notices
Imprint
- Terms + Terms
- Privacy + Privacy
@@ -300,7 +300,7 @@
Resources
Links
- Discord + Discord beaconcha.in GitHub Explorer GitHub Mobile App diff --git a/templates/login.html b/templates/login.html index d6060c8173..7aaa4a6150 100644 --- a/templates/login.html +++ b/templates/login.html @@ -39,7 +39,7 @@ {{ end }} {{ define "content" }} - {{ with .Data }} + {{ with .Data.AuthData }}
@@ -68,6 +68,8 @@

Sign in to beaconcha.in

+ + Don't have an account? Sign up diff --git a/templates/mail/layout.html b/templates/mail/layout.html index 68f4cca386..ae63b9f367 100644 --- a/templates/mail/layout.html +++ b/templates/mail/layout.html @@ -50,7 +50,7 @@

- Join our discord server for questions and feedback - https://discord.io/beaconchain + Join our discord server for questions and feedback - https://dsc.gg/beaconchain diff --git a/templates/payment/pricing.html b/templates/payment/pricing.html index b9005edc77..dfc99cfe89 100644 --- a/templates/payment/pricing.html +++ b/templates/payment/pricing.html @@ -202,17 +202,21 @@

Free

0€ / mo

    +
  • + Rate limit per second:
    + {{ formatAddCommas 5 }} +
  • Rate limit per minute:
    - 10 + {{ formatAddCommas 20 }}
  • Requests per day:
    - 10,000 + {{ formatAddCommas 30000 }}
  • Total requests per month:
    - 30,000 + {{ formatAddCommas 30000 }}
Get started @@ -225,17 +229,21 @@

Sapphire Popul

59€* / mo

    +
  • + Rate limit per second:
    + {{ formatAddCommas 10 }} +
  • Rate limit per minute:
    - 100 + {{ formatAddCommas 100 }}
  • Requests per day:
    - 100,000 + {{ formatAddCommas 100000 }}
  • Total requests per month:
    - 500,000 + {{ formatAddCommas 500000 }}
{{ if .User.Authenticated }} @@ -266,17 +274,21 @@

Emerald

99€* / mo

    +
  • + Rate limit per second:
    + {{ formatAddCommas 10 }} +
  • Rate limit per minute:
    - Unlimited + {{ formatAddCommas 600 }}
  • Requests per day:
    - 200,000 + {{ formatAddCommas 200000 }}
  • Total requests per month:
    - 1,000,000 + {{ formatAddCommas 1000000 }}
{{ if .User.Authenticated }} @@ -304,9 +316,13 @@

Diamond

399€* / mo

    +
  • + Rate limit per second:
    + {{ formatAddCommas 30 }} +
  • Rate limit per minute:
    - Unlimited + {{ formatAddCommas 1200 }}
  • Requests per day:
    @@ -314,7 +330,7 @@

    399€* / mo

  • Total requests per month:
    - 4,000,000 + {{ formatAddCommas 6000000 }}
{{ if .User.Authenticated }} @@ -378,6 +394,7 @@

Contact us about your API needs

+
diff --git a/templates/poap.html b/templates/poap.html deleted file mode 100644 index 1d0c91c5a1..0000000000 --- a/templates/poap.html +++ /dev/null @@ -1,448 +0,0 @@ -{{ define "js" }} - - - -{{ end }} - -{{ define "css" }} - - -{{ end }} - -{{ define "content" }} - {{ with .Data }} -
-
-
-

POAP - Proof of Attendance Protocol

- -
-
-
-
-

To celebrate the much anticipated launch of the Ethereum beacon chain, the ETHStakers community with assistance of Proof of Attendance Protocol are issuing a set of special NFTs. There are 3 different tokens:

-

- There are 3 different tokens -

-

Genesis depositor: Anyone who deposits 32 ETH on the deposit contract.

-

Block Proposer 1 - 1024: Validators that propose one of the first 1024 blocks and have to have an encoded ETH address on the graffiti field

-

Block Proposer 1 - 32768: Validators that propose one of the the first 32768 blocks and have to have an encoded ETH address on the graffiti field.

-

What is POAP?

-

The Proof of Attendance Protocol is a dapp that distributes badges in the form of ERC-721 tokens to prove you participated in an event. More info on the dapp and the multiclient token can be found here.

-

How will it work?

- -
    -
  1. Install an Ethereum consensus & execution layer client and set up a node and validator.
    2. -
  2. Add the Ethereum (Mainnet) address you want to receive the badge within the field provided.
    3. -
  3. Add the generated graffiti flag to your validator client.
    4. -
  4. - If you propose a block between genesis and block - 32768 - , your Eth address will be recorded on the beacon-chain, and you will be able to claim a token with that address using POAPs website. -
    5. -
  5. The tokens will be distributed via a mechanism called a merkle drop. This is like an airdrop, but badges are only distributed to those who actively claim them reducing state bloat and chain congestion.
    6. -
- -

Generate the flag for you validator:

- -
-
- - - Mega badge - Created with Sketch. - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
-
-
- - - -
-
-
-
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
ClientGraffiti
- Prysm - - - - - -
- Lighthouse - - - - - -
- Teku - - - - - -
- Nimbus - - - - - -
- Lodestar - - - - - -
-
-
-
-
-
-
-
-

Ethereum Mainnet Leaderboard

- -
-
-
- - - - - - {{ range $i, $client := .PoapClients }} - - {{ end }} - - - - - {{ range $i, $client := .PoapClients }} - - - {{ end }} - - - -
ETH AddressTotal{{ $client }}
BlocksValidatorsBlocksValidators
-
-
-
-
- {{ end }} -{{ end }} diff --git a/templates/slots.html b/templates/slots.html index 4bb371f2ce..5f7afe630b 100644 --- a/templates/slots.html +++ b/templates/slots.html @@ -1,6 +1,7 @@ {{ define "js" }} + - - -{{ end }} - -{{ define "css" }} - -{{ end }} - -{{ define "content" }} - {{ with .Data }} -
-
-
-

Attestation Streak Leaderboard

- - -
- Streak represents the amount of attestations that have been proposed in a row. If a validator misses an attestation, the current length will reset to 0 -
-
-
-
- - - - - - - - - - - - - - - - - -
IndexCurrent StreakLongest Streak
RankStartLengthRankStartLength
-
-
-
-
-
- {{ end }} -{{ end }} diff --git a/templates/withdrawals.html b/templates/withdrawals.html index 23a4206c4d..5e857cb028 100644 --- a/templates/withdrawals.html +++ b/templates/withdrawals.html @@ -213,7 +213,7 @@

{{ template "withdrawalChart" . }}

-
This table displays partial and full withdrawals.
+
This table displays partial and full withdrawals. Only the first 10000 entries of the resultset are shown.
@@ -239,7 +239,7 @@

Address Changes (BLS)

-
This table displays the BLS address changes from 0x00 credentials to 0x01.
+
This table displays the BLS address changes from 0x00 credentials to 0x01. Only the first 10000 entries of the resultset are shown.
diff --git a/types/api.go b/types/api.go index a7a6ecd1bb..86e6f0886d 100644 --- a/types/api.go +++ b/types/api.go @@ -715,3 +715,10 @@ type EnsDomainResponse struct { Address string `json:"address"` Domain string `json:"domain"` } + +type ApiProposalLuckResponse struct { + ProposalLuck *float64 `json:"proposal_luck"` // The proposal luck for the given set of validators as a percentage + AverageProposalInterval float64 `json:"average_proposal_interval"` // The average slot interval between proposals for the given set of validators + NextProposalEstimateTs *int64 `json:"next_proposal_estimate_ts"` // The estimated timestamp of the next proposal + TimeFrameName *string `json:"time_frame_name"` // The timeframe for which the luck is calculated +} diff --git a/types/config.go b/types/config.go index 111dab3479..8a9df8f8a8 100644 --- a/types/config.go +++ b/types/config.go @@ -28,8 +28,11 @@ type Config struct { MaxIdleConns int `yaml:"maxIdleConns" envconfig:"WRITER_DB_MAX_IDLE_CONNS"` } `yaml:"writerDatabase"` Bigtable struct { - Project string `yaml:"project" envconfig:"BIGTABLE_PROJECT"` - Instance string `yaml:"instance" envconfig:"BIGTABLE_INSTANCE"` + Project string `yaml:"project" envconfig:"BIGTABLE_PROJECT"` + Instance string `yaml:"instance" envconfig:"BIGTABLE_INSTANCE"` + Emulator bool `yaml:"emulator" envconfig:"BIGTABLE_EMULATOR"` + EmulatorPort int `yaml:"emulatorPort" envconfig:"BIGTABLE_EMULATOR_PORT"` + EmulatorHost string `yaml:"emulatorHost" envconfig:"BIGTABLE_EMULATOR_HOST"` } `yaml:"bigtable"` BlobIndexer struct { S3 struct { @@ -97,8 +100,12 @@ type Config struct { Enabled bool `yaml:"enabled" envconfig:"FRONTEND_ENABLED"` BlobProviderUrl string `yaml:"blobProviderUrl" envconfig:"FRONTEND_BLOB_PROVIDER_URL"` // Imprint is deprdecated place imprint file into the legal directory - Imprint string `yaml:"imprint" envconfig:"FRONTEND_IMPRINT"` - LegalDir string `yaml:"legalDir" envconfig:"FRONTEND_LEGAL"` + Imprint string `yaml:"imprint" envconfig:"FRONTEND_IMPRINT"` + Legal struct { + TermsOfServiceUrl string `yaml:"termsOfServiceUrl" envconfig:"FRONTEND_LEGAL_TERMS_OF_SERVICE_URL"` + PrivacyPolicyUrl string `yaml:"privacyPolicyUrl" envconfig:"FRONTEND_LEGAL_PRIVACY_POLICY_URL"` + ImprintTemplate string `yaml:"imprintTemplate" envconfig:"FRONTEND_LEGAL_IMPRINT_TEMPLATE"` + } `yaml:"legal"` SiteDomain string `yaml:"siteDomain" envconfig:"FRONTEND_SITE_DOMAIN"` SiteName string `yaml:"siteName" envconfig:"FRONTEND_SITE_NAME"` SiteSubtitle string `yaml:"siteSubtitle" envconfig:"FRONTEND_SITE_SUBTITLE"` @@ -168,9 +175,6 @@ type Config struct { Timestamp uint64 `yaml:"timestamp" envconfig:"FRONTEND_COUNTDOWN_TIMESTAMP"` Info string `yaml:"info" envconfig:"FRONTEND_COUNTDOWN_INFO"` } `yaml:"countdown"` - Validator struct { - ShowProposerRewards bool `yaml:"showProposerRewards" envconfig:"FRONTEND_SHOW_PROPOSER_REWARDS"` - } `yaml:"validator"` HttpReadTimeout time.Duration `yaml:"httpReadTimeout" envconfig:"FRONTEND_HTTP_READ_TIMEOUT"` HttpWriteTimeout time.Duration `yaml:"httpWriteTimeout" envconfig:"FRONTEND_HTTP_WRITE_TIMEOUT"` HttpIdleTimeout time.Duration `yaml:"httpIdleTimeout" envconfig:"FRONTEND_HTTP_IDLE_TIMEOUT"` @@ -229,3 +233,103 @@ type ServiceMonitoringConfiguration struct { Name string `yaml:"name" envconfig:"NAME"` Duration time.Duration `yaml:"duration" envconfig:"DURATION"` } + +type ConfigJsonResponse struct { + Data struct { + ConfigName string `json:"CONFIG_NAME"` + PresetBase string `json:"PRESET_BASE"` + TerminalTotalDifficulty string `json:"TERMINAL_TOTAL_DIFFICULTY"` + TerminalBlockHash string `json:"TERMINAL_BLOCK_HASH"` + TerminalBlockHashActivationEpoch string `json:"TERMINAL_BLOCK_HASH_ACTIVATION_EPOCH"` + SafeSlotsToImportOptimistically string `json:"SAFE_SLOTS_TO_IMPORT_OPTIMISTICALLY"` + MinGenesisActiveValidatorCount string `json:"MIN_GENESIS_ACTIVE_VALIDATOR_COUNT"` + MinGenesisTime string `json:"MIN_GENESIS_TIME"` + GenesisForkVersion string `json:"GENESIS_FORK_VERSION"` + GenesisDelay string `json:"GENESIS_DELAY"` + AltairForkVersion string `json:"ALTAIR_FORK_VERSION"` + AltairForkEpoch string `json:"ALTAIR_FORK_EPOCH"` + BellatrixForkVersion string `json:"BELLATRIX_FORK_VERSION"` + BellatrixForkEpoch string `json:"BELLATRIX_FORK_EPOCH"` + CapellaForkVersion string `json:"CAPELLA_FORK_VERSION"` + CapellaForkEpoch string `json:"CAPELLA_FORK_EPOCH"` + SecondsPerSlot string `json:"SECONDS_PER_SLOT"` + SecondsPerEth1Block string `json:"SECONDS_PER_ETH1_BLOCK"` + MinValidatorWithdrawabilityDelay string `json:"MIN_VALIDATOR_WITHDRAWABILITY_DELAY"` + ShardCommitteePeriod string `json:"SHARD_COMMITTEE_PERIOD"` + Eth1FollowDistance string `json:"ETH1_FOLLOW_DISTANCE"` + SubnetsPerNode string `json:"SUBNETS_PER_NODE"` + InactivityScoreBias string `json:"INACTIVITY_SCORE_BIAS"` + InactivityScoreRecoveryRate string `json:"INACTIVITY_SCORE_RECOVERY_RATE"` + EjectionBalance string `json:"EJECTION_BALANCE"` + MinPerEpochChurnLimit string `json:"MIN_PER_EPOCH_CHURN_LIMIT"` + ChurnLimitQuotient string `json:"CHURN_LIMIT_QUOTIENT"` + ProposerScoreBoost string `json:"PROPOSER_SCORE_BOOST"` + DepositChainID string `json:"DEPOSIT_CHAIN_ID"` + DepositNetworkID string `json:"DEPOSIT_NETWORK_ID"` + DepositContractAddress string `json:"DEPOSIT_CONTRACT_ADDRESS"` + MaxCommitteesPerSlot string `json:"MAX_COMMITTEES_PER_SLOT"` + TargetCommitteeSize string `json:"TARGET_COMMITTEE_SIZE"` + MaxValidatorsPerCommittee string `json:"MAX_VALIDATORS_PER_COMMITTEE"` + ShuffleRoundCount string `json:"SHUFFLE_ROUND_COUNT"` + HysteresisQuotient string `json:"HYSTERESIS_QUOTIENT"` + HysteresisDownwardMultiplier string `json:"HYSTERESIS_DOWNWARD_MULTIPLIER"` + HysteresisUpwardMultiplier string `json:"HYSTERESIS_UPWARD_MULTIPLIER"` + SafeSlotsToUpdateJustified string `json:"SAFE_SLOTS_TO_UPDATE_JUSTIFIED"` + MinDepositAmount string `json:"MIN_DEPOSIT_AMOUNT"` + MaxEffectiveBalance string `json:"MAX_EFFECTIVE_BALANCE"` + EffectiveBalanceIncrement string `json:"EFFECTIVE_BALANCE_INCREMENT"` + MinAttestationInclusionDelay string `json:"MIN_ATTESTATION_INCLUSION_DELAY"` + SlotsPerEpoch string `json:"SLOTS_PER_EPOCH"` + MinSeedLookahead string `json:"MIN_SEED_LOOKAHEAD"` + MaxSeedLookahead string `json:"MAX_SEED_LOOKAHEAD"` + EpochsPerEth1VotingPeriod string `json:"EPOCHS_PER_ETH1_VOTING_PERIOD"` + SlotsPerHistoricalRoot string `json:"SLOTS_PER_HISTORICAL_ROOT"` + MinEpochsToInactivityPenalty string `json:"MIN_EPOCHS_TO_INACTIVITY_PENALTY"` + EpochsPerHistoricalVector string `json:"EPOCHS_PER_HISTORICAL_VECTOR"` + EpochsPerSlashingsVector string `json:"EPOCHS_PER_SLASHINGS_VECTOR"` + HistoricalRootsLimit string `json:"HISTORICAL_ROOTS_LIMIT"` + ValidatorRegistryLimit string `json:"VALIDATOR_REGISTRY_LIMIT"` + BaseRewardFactor string `json:"BASE_REWARD_FACTOR"` + WhistleblowerRewardQuotient string `json:"WHISTLEBLOWER_REWARD_QUOTIENT"` + ProposerRewardQuotient string `json:"PROPOSER_REWARD_QUOTIENT"` + InactivityPenaltyQuotient string `json:"INACTIVITY_PENALTY_QUOTIENT"` + MinSlashingPenaltyQuotient string `json:"MIN_SLASHING_PENALTY_QUOTIENT"` + ProportionalSlashingMultiplier string `json:"PROPORTIONAL_SLASHING_MULTIPLIER"` + MaxProposerSlashings string `json:"MAX_PROPOSER_SLASHINGS"` + MaxAttesterSlashings string `json:"MAX_ATTESTER_SLASHINGS"` + MaxAttestations string `json:"MAX_ATTESTATIONS"` + MaxDeposits string `json:"MAX_DEPOSITS"` + MaxVoluntaryExits string `json:"MAX_VOLUNTARY_EXITS"` + InactivityPenaltyQuotientAltair string `json:"INACTIVITY_PENALTY_QUOTIENT_ALTAIR"` + MinSlashingPenaltyQuotientAltair string `json:"MIN_SLASHING_PENALTY_QUOTIENT_ALTAIR"` + ProportionalSlashingMultiplierAltair string `json:"PROPORTIONAL_SLASHING_MULTIPLIER_ALTAIR"` + SyncCommitteeSize string `json:"SYNC_COMMITTEE_SIZE"` + EpochsPerSyncCommitteePeriod string `json:"EPOCHS_PER_SYNC_COMMITTEE_PERIOD"` + MinSyncCommitteeParticipants string `json:"MIN_SYNC_COMMITTEE_PARTICIPANTS"` + InactivityPenaltyQuotientBellatrix string `json:"INACTIVITY_PENALTY_QUOTIENT_BELLATRIX"` + MinSlashingPenaltyQuotientBellatrix string `json:"MIN_SLASHING_PENALTY_QUOTIENT_BELLATRIX"` + ProportionalSlashingMultiplierBellatrix string `json:"PROPORTIONAL_SLASHING_MULTIPLIER_BELLATRIX"` + MaxBytesPerTransaction string `json:"MAX_BYTES_PER_TRANSACTION"` + MaxTransactionsPerPayload string `json:"MAX_TRANSACTIONS_PER_PAYLOAD"` + BytesPerLogsBloom string `json:"BYTES_PER_LOGS_BLOOM"` + MaxExtraDataBytes string `json:"MAX_EXTRA_DATA_BYTES"` + MaxBlsToExecutionChanges string `json:"MAX_BLS_TO_EXECUTION_CHANGES"` + MaxWithdrawalsPerPayload string `json:"MAX_WITHDRAWALS_PER_PAYLOAD"` + MaxValidatorsPerWithdrawalsSweep string `json:"MAX_VALIDATORS_PER_WITHDRAWALS_SWEEP"` + DomainAggregateAndProof string `json:"DOMAIN_AGGREGATE_AND_PROOF"` + TargetAggregatorsPerSyncSubcommittee string `json:"TARGET_AGGREGATORS_PER_SYNC_SUBCOMMITTEE"` + SyncCommitteeSubnetCount string `json:"SYNC_COMMITTEE_SUBNET_COUNT"` + BlsWithdrawalPrefix string `json:"BLS_WITHDRAWAL_PREFIX"` + DomainRandao string `json:"DOMAIN_RANDAO"` + DomainVoluntaryExit string `json:"DOMAIN_VOLUNTARY_EXIT"` + DomainSyncCommitteeSelectionProof string `json:"DOMAIN_SYNC_COMMITTEE_SELECTION_PROOF"` + DomainBeaconAttester string `json:"DOMAIN_BEACON_ATTESTER"` + DomainBeaconProposer string `json:"DOMAIN_BEACON_PROPOSER"` + DomainDeposit string `json:"DOMAIN_DEPOSIT"` + DomainSelectionProof string `json:"DOMAIN_SELECTION_PROOF"` + DomainSyncCommittee string `json:"DOMAIN_SYNC_COMMITTEE"` + TargetAggregatorsPerCommittee string `json:"TARGET_AGGREGATORS_PER_COMMITTEE"` + DomainContributionAndProof string `json:"DOMAIN_CONTRIBUTION_AND_PROOF"` + DomainApplicationMask string `json:"DOMAIN_APPLICATION_MASK"` + } `json:"data"` +} diff --git a/types/exporter.go b/types/exporter.go index d6fb944d88..99ea9bc86f 100644 --- a/types/exporter.go +++ b/types/exporter.go @@ -45,13 +45,20 @@ type FinalityCheckpoints struct { } `json:"finalized"` } +type Slot uint64 +type ValidatorIndex uint64 + // EpochData is a struct to hold epoch data type EpochData struct { Epoch uint64 Validators []*Validator ValidatorAssignmentes *EpochAssignments Blocks map[uint64]map[string]*Block + FutureBlocks map[uint64]map[string]*Block EpochParticipationStats *ValidatorParticipation + AttestationDuties map[Slot]map[ValidatorIndex][]Slot + SyncDuties map[Slot]map[ValidatorIndex]bool + Finalized bool } // ValidatorParticipation is a struct to hold validator participation data @@ -60,6 +67,7 @@ type ValidatorParticipation struct { GlobalParticipationRate float32 VotedEther uint64 EligibleEther uint64 + Finalized bool } // BeaconCommitteItem is a struct to hold beacon committee data @@ -127,6 +135,8 @@ type Block struct { ExcessBlobGas uint64 BlobKZGCommitments [][]byte BlobKZGProofs [][]byte + AttestationDuties map[ValidatorIndex]Slot + SyncDuties map[ValidatorIndex]bool } type SignedBLSToExecutionChange struct { diff --git a/types/templates.go b/types/templates.go index 14a8af6b3e..7cc5d12841 100644 --- a/types/templates.go +++ b/types/templates.go @@ -52,6 +52,8 @@ type PageData struct { GlobalNotification template.HTML AvailableCurrencies []string MainMenuItems []MainMenuItem + TermsOfServiceUrl string + PrivacyPolicyUrl string } type MainMenuItem struct { @@ -377,17 +379,16 @@ type ValidatorPageData struct { ParticipatedSyncCountSlots uint64 MissedSyncCountSlots uint64 OrphanedSyncCountSlots uint64 - UnmissedSyncPercentage float64 // participated/(participated+missed) - IncomeToday ClElInt64 `json:"incomeToday"` - Income1d ClElInt64 `json:"income1d"` - Income7d ClElInt64 `json:"income7d"` - Income31d ClElInt64 `json:"income31d"` - IncomeTotal ClElInt64 `json:"incomeTotal"` - IncomeTotalFormatted template.HTML `json:"incomeTotalFormatted"` - IncomeProposerFormatted *template.HTML `json:"incomeProposerFormatted"` - Apr7d ClElFloat64 `json:"apr7d"` - Apr31d ClElFloat64 `json:"apr31d"` - Apr365d ClElFloat64 `json:"apr365d"` + UnmissedSyncPercentage float64 // participated/(participated+missed) + IncomeToday ClElInt64 `json:"incomeToday"` + Income1d ClElInt64 `json:"income1d"` + Income7d ClElInt64 `json:"income7d"` + Income31d ClElInt64 `json:"income31d"` + IncomeTotal ClElInt64 `json:"incomeTotal"` + IncomeTotalFormatted template.HTML `json:"incomeTotalFormatted"` + Apr7d ClElFloat64 `json:"apr7d"` + Apr31d ClElFloat64 `json:"apr31d"` + Apr365d ClElFloat64 `json:"apr365d"` SyncLuck float64 SyncEstimate *time.Time AvgSyncInterval *time.Duration @@ -1058,7 +1059,6 @@ type ValidatorEarnings struct { LastDayFormatted template.HTML `json:"lastDayFormatted"` LastWeekFormatted template.HTML `json:"lastWeekFormatted"` LastMonthFormatted template.HTML `json:"lastMonthFormatted"` - ProposerTotalFormatted template.HTML `json:"proposerTotalFormatted"` TotalFormatted template.HTML `json:"totalFormatted"` TotalChangeFormatted template.HTML `json:"totalChangeFormatted"` TotalBalance template.HTML `json:"totalBalance"` @@ -1686,6 +1686,11 @@ type DepositContractInteraction struct { Amount []byte } +type EpochInfo struct { + Finalized bool `db:"finalized"` + Participation float64 `db:"globalparticipationrate"` +} + type Eth1TxData struct { From common.Address To *common.Address @@ -1705,10 +1710,7 @@ type Eth1TxData struct { BlobGasPrice []byte BlobFee []byte } - Epoch struct { - Finalized bool `db:"finalized"` - Participation float64 `db:"globalparticipationrate"` - } + Epoch EpochInfo TypeFormatted string Type uint8 Nonce uint64 @@ -2088,17 +2090,16 @@ type BroadcastStatusPageData struct { } type ValidatorIncomePerformance struct { - ClIncome1d int64 `db:"cl_performance_1d"` - ClIncome7d int64 `db:"cl_performance_7d"` - ClIncome31d int64 `db:"cl_performance_31d"` - ClIncome365d int64 `db:"cl_performance_365d"` - ClIncomeTotal int64 `db:"cl_performance_total"` - ClProposerIncomeTotal int64 `db:"cl_proposer_performance_total"` - ElIncome1d int64 `db:"el_performance_1d"` - ElIncome7d int64 `db:"el_performance_7d"` - ElIncome31d int64 `db:"el_performance_31d"` - ElIncome365d int64 `db:"el_performance_365d"` - ElIncomeTotal int64 `db:"el_performance_total"` + ClIncome1d int64 `db:"cl_performance_1d"` + ClIncome7d int64 `db:"cl_performance_7d"` + ClIncome31d int64 `db:"cl_performance_31d"` + ClIncome365d int64 `db:"cl_performance_365d"` + ClIncomeTotal int64 `db:"cl_performance_total"` + ElIncome1d int64 `db:"el_performance_1d"` + ElIncome7d int64 `db:"el_performance_7d"` + ElIncome31d int64 `db:"el_performance_31d"` + ElIncome365d int64 `db:"el_performance_365d"` + ElIncomeTotal int64 `db:"el_performance_total"` } type ValidatorProposalInfo struct { diff --git a/utils/eth1.go b/utils/eth1.go index 905f3ccd14..4a83877b15 100644 --- a/utils/eth1.go +++ b/utils/eth1.go @@ -261,14 +261,15 @@ func FormatAddressAsTokenLink(token, address []byte, name string, verified bool, func FormatHashLong(hash common.Hash) template.HTML { address := hash.String() - test := ` -
- %s - %s - %s -
` if len(address) > 4 { - return template.HTML(fmt.Sprintf(test, address[:4], address[4:len(address)-4], address[len(address)-4:])) + htmlFormat := ` +
+ %s + %s + %s +
` + + return template.HTML(fmt.Sprintf(htmlFormat, address[:4], address[4:len(address)-4], address[len(address)-4:])) } return template.HTML(address) @@ -279,10 +280,11 @@ func FormatAddressLong(address string) template.HTML { return template.HTML(address) } address = FixAddressCasing(address) - test := ` - %s%s%s` if len(address) > 4 { - return template.HTML(fmt.Sprintf(test, address[:6], address[6:len(address)-4], address[len(address)-4:])) + htmlFormat := ` + %s%s%s%s` + + return template.HTML(fmt.Sprintf(htmlFormat, address[:2], address[2:6], address[6:len(address)-4], address[len(address)-4:])) } return template.HTML(address) diff --git a/utils/oauthutils.go b/utils/oauthutils.go index 89364be64e..8407f678f3 100644 --- a/utils/oauthutils.go +++ b/utils/oauthutils.go @@ -5,7 +5,7 @@ import ( "encoding/json" "errors" "fmt" - "io/ioutil" + "io" "net/http" "strings" "time" @@ -241,7 +241,7 @@ func AuthorizedAPIMiddleware(next http.Handler) http.Handler { // as context to the request. By doing this here, // we can use base.FormValueOrJson(key) without multiple parsings of the same body if r.Method == "POST" && r.Header.Get("Content-Type") == "application/json" { - body, err := ioutil.ReadAll(r.Body) + body, err := io.ReadAll(r.Body) if err == nil { keyVal := make(map[string]interface{}) json.Unmarshal(body, &keyVal) diff --git a/utils/utils.go b/utils/utils.go index 99a06db6d0..7bbe59217f 100644 --- a/utils/utils.go +++ b/utils/utils.go @@ -18,7 +18,6 @@ import ( "html/template" "image/color" "io" - "io/ioutil" "log" "math" "math/big" @@ -30,6 +29,7 @@ import ( "regexp" "runtime" "sort" + "strconv" "strings" "time" "unicode/utf8" @@ -40,6 +40,7 @@ import ( "gopkg.in/yaml.v3" "github.com/asaskevich/govalidator" + "github.com/carlmjohnson/requests" "github.com/ethereum/go-ethereum/accounts/abi" "github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethereum/go-ethereum/params" @@ -246,7 +247,7 @@ func GetTemplateFuncs() template.FuncMap { // IncludeHTML adds html to the page func IncludeHTML(path string) template.HTML { - b, err := ioutil.ReadFile(path) + b, err := os.ReadFile(path) if err != nil { log.Printf("includeHTML - error reading file: %v", err) return "" @@ -254,9 +255,9 @@ func IncludeHTML(path string) template.HTML { return template.HTML(string(b)) } -func GraffitiToSring(graffiti []byte) string { +func GraffitiToString(graffiti []byte) string { s := strings.Map(fixUtf, string(bytes.Trim(graffiti, "\x00"))) - s = strings.Replace(s, "\u0000", "", -1) // rempove 0x00 bytes as it is not supported in postgres + s = strings.Replace(s, "\u0000", "", -1) // remove 0x00 bytes as it is not supported in postgres if !utf8.ValidString(s) { return "INVALID_UTF8_STRING" @@ -270,6 +271,22 @@ func FormatGraffitiString(graffiti string) string { return strings.Map(fixUtf, template.HTMLEscapeString(graffiti)) } +func HasProblematicUtfCharacters(s string) bool { + // Check for null character ('\x00') + if utf8.ValidString(s) && utf8.Valid([]byte(s)) { + // Check for control characters ('\x01' to '\x1F' and '\x7F') + for _, r := range s { + if r <= 0x1F || r == 0x7F { + return true + } + } + } else { + return true // Invalid UTF-8 sequence + } + + return false +} + func fixUtf(r rune) rune { if r == utf8.RuneError { return -1 @@ -373,6 +390,7 @@ func WaitForCtrlC() { <-c } +/* func ReadClConfig(clCfg *types.ClChainConfig, name string, cfgPath string) error { // first check the preset and set all values from it var cfgBytes []byte @@ -441,10 +459,16 @@ func ReadClConfig(clCfg *types.ClChainConfig, name string, cfgPath string) error err = yaml.Unmarshal(cfgBytes, &clCfg) return err } +*/ // ReadConfig will process a configuration func ReadConfig(cfg *types.Config, path string) error { + configPathFromEnv := os.Getenv("BEACONCHAIN_CONFIG") + + if configPathFromEnv != "" { // allow the location of the config file to be passed via env args + path = configPathFromEnv + } if strings.HasPrefix(path, "projects/") { x, err := AccessSecretVersion(path) if err != nil { @@ -470,34 +494,166 @@ func ReadConfig(cfg *types.Config, path string) error { return err } - err = ReadClConfig(&cfg.Chain.ClConfig, cfg.Chain.Name, cfg.Chain.ClConfigPath) - if err != nil { - return err - } + if cfg.Chain.ClConfigPath == "" { + // var prysmParamsConfig *prysmParams.BeaconChainConfig + switch cfg.Chain.Name { + case "mainnet": + err = yaml.Unmarshal([]byte(config.MainnetChainYml), &cfg.Chain.ClConfig) + case "prater": + err = yaml.Unmarshal([]byte(config.PraterChainYml), &cfg.Chain.ClConfig) + case "ropsten": + err = yaml.Unmarshal([]byte(config.RopstenChainYml), &cfg.Chain.ClConfig) + case "sepolia": + err = yaml.Unmarshal([]byte(config.SepoliaChainYml), &cfg.Chain.ClConfig) + case "gnosis": + err = yaml.Unmarshal([]byte(config.GnosisChainYml), &cfg.Chain.ClConfig) + case "holesky": + err = yaml.Unmarshal([]byte(config.HoleskyChainYml), &cfg.Chain.ClConfig) + default: + return fmt.Errorf("tried to set known chain-config, but unknown chain-name") + } + if err != nil { + return err + } + // err = prysmParams.SetActive(prysmParamsConfig) + // if err != nil { + // return fmt.Errorf("error setting chainConfig (%v) for prysmParams: %w", cfg.Chain.Name, err) + // } + } else if cfg.Chain.ClConfigPath == "node" { + nodeEndpoint := fmt.Sprintf("http://%s:%s", cfg.Indexer.Node.Host, cfg.Indexer.Node.Port) - switch cfg.Chain.Name { - case "mainnet": - cfg.Chain.ElConfig = params.MainnetChainConfig - case "prater": - cfg.Chain.ElConfig = params.GoerliChainConfig - case "sepolia": - cfg.Chain.ElConfig = params.SepoliaChainConfig - case "gnosis": - // #TODO.patrick - cfg.Chain.ElConfig = ¶ms.ChainConfig{} - case "dencun-devnet-8": - // https://raw.githubusercontent.com/ethpandaops/dencun-testnet/master/network-configs/devnet-8/genesis.json - // #TODO.patrick - cfg.Chain.ElConfig = ¶ms.ChainConfig{} - cfg.Chain.ElConfig.ChainID = new(big.Int).SetInt64(7011893058) - cfg.Chain.ElConfig.TerminalTotalDifficultyPassed = true - shanghaiTime := uint64(1692184320) - cfg.Chain.ElConfig.ShanghaiTime = &shanghaiTime - cancunTime := uint64(1692186240) - cfg.Chain.ElConfig.CancunTime = &cancunTime - default: - cfg.Chain.ElConfig = ¶ms.ChainConfig{} + jr := &types.ConfigJsonResponse{} + + err := requests. + URL(nodeEndpoint + "/eth/v1/config/spec"). + ToJSON(jr). + Fetch(context.Background()) + + if err != nil { + return err + } + + chainCfg := types.ClChainConfig{ + PresetBase: jr.Data.PresetBase, + ConfigName: jr.Data.ConfigName, + TerminalTotalDifficulty: jr.Data.TerminalTotalDifficulty, + TerminalBlockHash: jr.Data.TerminalBlockHash, + TerminalBlockHashActivationEpoch: mustParseUint(jr.Data.TerminalBlockHashActivationEpoch), + MinGenesisActiveValidatorCount: mustParseUint(jr.Data.MinGenesisActiveValidatorCount), + MinGenesisTime: int64(mustParseUint(jr.Data.MinGenesisTime)), + GenesisForkVersion: jr.Data.GenesisForkVersion, + GenesisDelay: mustParseUint(jr.Data.GenesisDelay), + AltairForkVersion: jr.Data.AltairForkVersion, + AltairForkEpoch: mustParseUint(jr.Data.AltairForkEpoch), + BellatrixForkVersion: jr.Data.BellatrixForkVersion, + BellatrixForkEpoch: mustParseUint(jr.Data.BellatrixForkEpoch), + CappellaForkVersion: jr.Data.CapellaForkVersion, + CappellaForkEpoch: mustParseUint(jr.Data.CapellaForkEpoch), + SecondsPerSlot: mustParseUint(jr.Data.SecondsPerSlot), + SecondsPerEth1Block: mustParseUint(jr.Data.SecondsPerEth1Block), + MinValidatorWithdrawabilityDelay: mustParseUint(jr.Data.MinValidatorWithdrawabilityDelay), + ShardCommitteePeriod: mustParseUint(jr.Data.ShardCommitteePeriod), + Eth1FollowDistance: mustParseUint(jr.Data.Eth1FollowDistance), + InactivityScoreBias: mustParseUint(jr.Data.InactivityScoreBias), + InactivityScoreRecoveryRate: mustParseUint(jr.Data.InactivityScoreRecoveryRate), + EjectionBalance: mustParseUint(jr.Data.EjectionBalance), + MinPerEpochChurnLimit: mustParseUint(jr.Data.MinPerEpochChurnLimit), + ChurnLimitQuotient: mustParseUint(jr.Data.ChurnLimitQuotient), + ProposerScoreBoost: mustParseUint(jr.Data.ProposerScoreBoost), + DepositChainID: mustParseUint(jr.Data.DepositChainID), + DepositNetworkID: mustParseUint(jr.Data.DepositNetworkID), + DepositContractAddress: jr.Data.DepositContractAddress, + MaxCommitteesPerSlot: mustParseUint(jr.Data.MaxCommitteesPerSlot), + TargetCommitteeSize: mustParseUint(jr.Data.TargetCommitteeSize), + MaxValidatorsPerCommittee: mustParseUint(jr.Data.TargetCommitteeSize), + ShuffleRoundCount: mustParseUint(jr.Data.ShuffleRoundCount), + HysteresisQuotient: mustParseUint(jr.Data.HysteresisQuotient), + HysteresisDownwardMultiplier: mustParseUint(jr.Data.HysteresisDownwardMultiplier), + HysteresisUpwardMultiplier: mustParseUint(jr.Data.HysteresisUpwardMultiplier), + SafeSlotsToUpdateJustified: mustParseUint(jr.Data.SafeSlotsToUpdateJustified), + MinDepositAmount: mustParseUint(jr.Data.MinDepositAmount), + MaxEffectiveBalance: mustParseUint(jr.Data.MaxEffectiveBalance), + EffectiveBalanceIncrement: mustParseUint(jr.Data.EffectiveBalanceIncrement), + MinAttestationInclusionDelay: mustParseUint(jr.Data.MinAttestationInclusionDelay), + SlotsPerEpoch: mustParseUint(jr.Data.SlotsPerEpoch), + MinSeedLookahead: mustParseUint(jr.Data.MinSeedLookahead), + MaxSeedLookahead: mustParseUint(jr.Data.MaxSeedLookahead), + EpochsPerEth1VotingPeriod: mustParseUint(jr.Data.EpochsPerEth1VotingPeriod), + SlotsPerHistoricalRoot: mustParseUint(jr.Data.SlotsPerHistoricalRoot), + MinEpochsToInactivityPenalty: mustParseUint(jr.Data.MinEpochsToInactivityPenalty), + EpochsPerHistoricalVector: mustParseUint(jr.Data.EpochsPerHistoricalVector), + EpochsPerSlashingsVector: mustParseUint(jr.Data.EpochsPerSlashingsVector), + HistoricalRootsLimit: mustParseUint(jr.Data.HistoricalRootsLimit), + ValidatorRegistryLimit: mustParseUint(jr.Data.ValidatorRegistryLimit), + BaseRewardFactor: mustParseUint(jr.Data.BaseRewardFactor), + WhistleblowerRewardQuotient: mustParseUint(jr.Data.WhistleblowerRewardQuotient), + ProposerRewardQuotient: mustParseUint(jr.Data.ProposerRewardQuotient), + InactivityPenaltyQuotient: mustParseUint(jr.Data.InactivityPenaltyQuotient), + MinSlashingPenaltyQuotient: mustParseUint(jr.Data.MinSlashingPenaltyQuotient), + ProportionalSlashingMultiplier: mustParseUint(jr.Data.ProportionalSlashingMultiplier), + MaxProposerSlashings: mustParseUint(jr.Data.MaxProposerSlashings), + MaxAttesterSlashings: mustParseUint(jr.Data.MaxAttesterSlashings), + MaxAttestations: mustParseUint(jr.Data.MaxAttestations), + MaxDeposits: mustParseUint(jr.Data.MaxDeposits), + MaxVoluntaryExits: mustParseUint(jr.Data.MaxVoluntaryExits), + InvactivityPenaltyQuotientAltair: mustParseUint(jr.Data.InactivityPenaltyQuotientAltair), + MinSlashingPenaltyQuotientAltair: mustParseUint(jr.Data.MinSlashingPenaltyQuotientAltair), + ProportionalSlashingMultiplierAltair: mustParseUint(jr.Data.ProportionalSlashingMultiplierAltair), + SyncCommitteeSize: mustParseUint(jr.Data.SyncCommitteeSize), + EpochsPerSyncCommitteePeriod: mustParseUint(jr.Data.EpochsPerSyncCommitteePeriod), + MinSyncCommitteeParticipants: mustParseUint(jr.Data.MinSyncCommitteeParticipants), + InvactivityPenaltyQuotientBellatrix: mustParseUint(jr.Data.InactivityPenaltyQuotientBellatrix), + MinSlashingPenaltyQuotientBellatrix: mustParseUint(jr.Data.MinSlashingPenaltyQuotientBellatrix), + ProportionalSlashingMultiplierBellatrix: mustParseUint(jr.Data.ProportionalSlashingMultiplierBellatrix), + MaxBytesPerTransaction: mustParseUint(jr.Data.MaxBytesPerTransaction), + MaxTransactionsPerPayload: mustParseUint(jr.Data.MaxTransactionsPerPayload), + BytesPerLogsBloom: mustParseUint(jr.Data.BytesPerLogsBloom), + MaxExtraDataBytes: mustParseUint(jr.Data.MaxExtraDataBytes), + MaxWithdrawalsPerPayload: mustParseUint(jr.Data.MaxWithdrawalsPerPayload), + MaxValidatorsPerWithdrawalSweep: mustParseUint(jr.Data.MaxValidatorsPerWithdrawalsSweep), + MaxBlsToExecutionChange: mustParseUint(jr.Data.MaxBlsToExecutionChanges), + } + + cfg.Chain.ClConfig = chainCfg + + type GenesisResponse struct { + Data struct { + GenesisTime string `json:"genesis_time"` + GenesisValidatorsRoot string `json:"genesis_validators_root"` + GenesisForkVersion string `json:"genesis_fork_version"` + } `json:"data"` + } + + gtr := &GenesisResponse{} + + err = requests. + URL(nodeEndpoint + "/eth/v1/beacon/genesis"). + ToJSON(gtr). + Fetch(context.Background()) + + if err != nil { + return err + } + + cfg.Chain.GenesisTimestamp = mustParseUint(gtr.Data.GenesisTime) + cfg.Chain.GenesisValidatorsRoot = gtr.Data.GenesisValidatorsRoot + + logger.Infof("loaded chain config from node with genesis time %s", gtr.Data.GenesisTime) + + } else { + f, err := os.Open(cfg.Chain.ClConfigPath) + if err != nil { + return fmt.Errorf("error opening Chain Config file %v: %w", cfg.Chain.ClConfigPath, err) + } + var chainConfig *types.ClChainConfig + decoder := yaml.NewDecoder(f) + err = decoder.Decode(&chainConfig) + if err != nil { + return fmt.Errorf("error decoding Chain Config file %v: %v", cfg.Chain.ClConfigPath, err) + } + cfg.Chain.ClConfig = *chainConfig } + cfg.Chain.Name = cfg.Chain.ClConfig.ConfigName if cfg.Chain.GenesisTimestamp == 0 { switch cfg.Chain.Name { @@ -507,10 +663,12 @@ func ReadConfig(cfg *types.Config, path string) error { cfg.Chain.GenesisTimestamp = 1616508000 case "sepolia": cfg.Chain.GenesisTimestamp = 1655733600 + case "zhejiang": + cfg.Chain.GenesisTimestamp = 1675263600 case "gnosis": cfg.Chain.GenesisTimestamp = 1638993340 - case "dencun-devnet-8": - cfg.Chain.GenesisTimestamp = 1692182400 + case "holesky": + cfg.Chain.GenesisTimestamp = 1694786400 default: return fmt.Errorf("tried to set known genesis-timestamp, but unknown chain-name") } @@ -524,10 +682,12 @@ func ReadConfig(cfg *types.Config, path string) error { cfg.Chain.GenesisValidatorsRoot = "0x043db0d9a83813551ee2f33450d23797757d430911a9320530ad8a0eabc43efb" case "sepolia": cfg.Chain.GenesisValidatorsRoot = "0xd8ea171f3c94aea21ebc42a1ed61052acf3f9209c00e4efbaaddac09ed9b8078" + case "zhejiang": + cfg.Chain.GenesisValidatorsRoot = "0x53a92d8f2bb1d85f62d16a156e6ebcd1bcaba652d0900b2c2f387826f3481f6f" case "gnosis": cfg.Chain.GenesisValidatorsRoot = "0xf5dcb5564e829aab27264b9becd5dfaa017085611224cb3036f573368dbb9d47" - case "dencun-devnet-8": - cfg.Chain.GenesisValidatorsRoot = "0x6079c0b803059b77610f0198ce4b4c459cf43afd4992359084894f719d40faba" + case "holesky": + cfg.Chain.GenesisValidatorsRoot = "0x9143aa7c615a7f7115e2b6aac319c03529df8242ae705fba9df39b79c59fa8b1" default: return fmt.Errorf("tried to set known genesis-validators-root, but unknown chain-name") } @@ -540,20 +700,6 @@ func ReadConfig(cfg *types.Config, path string) error { cfg.Chain.DomainVoluntaryExit = "0x04000000" } - if cfg.Chain.ClConfig.DepositChainID != cfg.Chain.ElConfig.ChainID.Uint64() { - logrus.Fatalf("cfg.Chain.ClConfig.DepositChainID != cfg.Chain.ElConfig.ChainID.Uint64(): %v != %v", cfg.Chain.ClConfig.DepositChainID, cfg.Chain.ElConfig.ChainID.Uint64()) - } - - // #TODO.patrick - if cfg.Chain.ClConfig.ConfigName == "dencun-devnet-8" { - if cfg.Chain.ClConfig.MinSlashingPenaltyQuotientAltair != 64 { - logrus.Fatal("cfg.Chain.ClConfig.MinSlashingPenaltyQuotientAltair != 64: %v", cfg.Chain.ClConfig.MinSlashingPenaltyQuotientAltair) - } - if cfg.Chain.ClConfig.GenesisForkVersion != "0x10109396" { - logrus.Fatal("cfg.Chain.ClConfig.GenesisForkVersion != 0x10109396: %v", cfg.Chain.ClConfig.GenesisForkVersion) - } - } - logrus.WithFields(logrus.Fields{ "genesisTimestamp": cfg.Chain.GenesisTimestamp, "genesisValidatorsRoot": cfg.Chain.GenesisValidatorsRoot, @@ -566,6 +712,20 @@ func ReadConfig(cfg *types.Config, path string) error { return nil } +func mustParseUint(str string) uint64 { + + if str == "" { + return 0 + } + + nbr, err := strconv.ParseUint(str, 10, 64) + if err != nil { + logrus.Fatalf("fatal error parsing uint %s: %v", str, err) + } + + return nbr +} + func readConfigFile(cfg *types.Config, path string) error { if path == "" { return yaml.Unmarshal([]byte(config.DefaultConfigYml), cfg) @@ -625,6 +785,7 @@ var withdrawalCredentialsRE = regexp.MustCompile("^(0x)?00[0-9a-fA-F]{62}$") var withdrawalCredentialsAddressRE = regexp.MustCompile("^(0x)?010000000000000000000000[0-9a-fA-F]{40}$") var eth1TxRE = regexp.MustCompile("^(0x)?[0-9a-fA-F]{64}$") var zeroHashRE = regexp.MustCompile("^(0x)?0+$") +var hashRE = regexp.MustCompile("^(0x)?[0-9a-fA-F]{96}$") // IsValidEth1Address verifies whether a string represents a valid eth1-address. func IsValidEth1Address(s string) bool { @@ -641,6 +802,11 @@ func IsValidEth1Tx(s string) bool { return !zeroHashRE.MatchString(s) && eth1TxRE.MatchString(s) } +// IsValidEth1Tx verifies whether a string represents a valid eth1-tx-hash. +func IsHash(s string) bool { + return hashRE.MatchString(s) +} + // IsValidWithdrawalCredentials verifies whether a string represents valid withdrawal credentials. func IsValidWithdrawalCredentials(s string) bool { return withdrawalCredentialsRE.MatchString(s) || withdrawalCredentialsAddressRE.MatchString(s) @@ -868,7 +1034,7 @@ func ValidateReCAPTCHA(recaptchaResponse string) (bool, error) { return false, err } defer req.Body.Close() - body, err := ioutil.ReadAll(req.Body) // Read the response from Google + body, err := io.ReadAll(req.Body) // Read the response from Google if err != nil { return false, err } @@ -929,7 +1095,7 @@ func TryFetchContractMetadata(address []byte) (*types.ContractMetadata, error) { // } // if resp.StatusCode == 200 { -// body, err := ioutil.ReadAll(resp.Body) +// body, err := io.ReadAll(resp.Body) // if err != nil { // return nil, err // } @@ -1004,7 +1170,7 @@ func getABIFromEtherscan(address []byte) (*types.ContractMetadata, error) { return nil, fmt.Errorf("StatusCode: '%d', Status: '%s'", resp.StatusCode, resp.Status) } - body, err := ioutil.ReadAll(resp.Body) + body, err := io.ReadAll(resp.Body) if err != nil { return nil, err } @@ -1345,7 +1511,7 @@ func GetRemainingScheduledSync(validatorCount int, stats types.SyncCommitteesSta // - `validators` : the validators to add the stats for // - `syncDutiesHistory` : the sync duties history of all queried validators // - `stats` : the stats object to add the stats to, if nil a new stats object is created -func AddSyncStats(validators []uint64, syncDutiesHistory map[uint64][]*types.ValidatorSyncParticipation, stats *types.SyncCommitteesStats) types.SyncCommitteesStats { +func AddSyncStats(validators []uint64, syncDutiesHistory map[uint64]map[uint64]*types.ValidatorSyncParticipation, stats *types.SyncCommitteesStats) types.SyncCommitteesStats { if stats == nil { stats = &types.SyncCommitteesStats{} } @@ -1470,7 +1636,7 @@ type HttpReqHttpError struct { } func (err *HttpReqHttpError) Error() string { - return fmt.Sprintf("urls: %s, status: %d, body: %s", err.Url, err.StatusCode, err.Body) + return fmt.Sprintf("error response: url: %s, status: %d, body: %s", err.Url, err.StatusCode, err.Body) } func HttpReq(ctx context.Context, method, url string, params, result interface{}) error { @@ -1479,7 +1645,7 @@ func HttpReq(ctx context.Context, method, url string, params, result interface{} if params != nil { paramsJSON, err := json.Marshal(params) if err != nil { - return fmt.Errorf("error marhsaling params for request: %w, url: %v", err, url) + return fmt.Errorf("error marshaling params for request: %w, url: %v", err, url) } req, err = http.NewRequestWithContext(ctx, method, url, bytes.NewBuffer(paramsJSON)) if err != nil { @@ -1514,3 +1680,16 @@ func HttpReq(ctx context.Context, method, url string, params, result interface{} } return nil } + +func ReverseString(s string) string { + runes := []rune(s) + for i, j := 0, len(runes)-1; i < j; i, j = i+1, j-1 { + runes[i], runes[j] = runes[j], runes[i] + } + return string(runes) +} + +func GetCurrentFuncName() string { + pc, _, _, _ := runtime.Caller(1) + return runtime.FuncForPC(pc).Name() +}