Skip to content

Commit

Permalink
Bids 2437/ens (#2919)
Browse files Browse the repository at this point in the history 
* (BIDS-2437) ens: debugging

* (BIDS-2437) handle another error when validating ens addresses

* (BIDS-2437) handle another error when validating ens addresses

* (BIDS-2437) handle another error when validating ens addresses

* (BIDS-2437) misc: fix db-config

* (BIDS-2437) eth1indexer: cleanup ens

* (BIDS-2437) make ens-indexing work with holesky and sepolia

* (BIDS-2437) make batchsize for ens-imports configurable

* (BIDS-2437) eth1indexer: add some metrics

* (BIDS-2437) eth1indexer: add flag metrics.addr
  • Loading branch information
@guybrush
guybrush authored Jul 22, 2024
1 parent 5967988 commit f879104
Show file tree
Hide file tree
Showing 27 changed files with 14,702 additions and 571 deletions.
24 changes: 23 additions & 1 deletion cmd/eth1indexer/main.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ import (
"encoding/json"
"eth2-exporter/db"
"eth2-exporter/erc20"
"eth2-exporter/metrics"
"eth2-exporter/rpc"
"eth2-exporter/services"
"eth2-exporter/types"
Expand Down Expand Up @@ -69,6 +70,9 @@ func main() {
configPath := flag.String("config", "", "Path to the config file, if empty string defaults will be used")

enableEnsUpdater := flag.Bool("ens.enabled", false, "Enable ens update process")
ensBatchSize := flag.Int64("ens.batch", 200, "Batch size for ens updates")

metricsAddr := flag.String("metrics.addr", "", "Metrics address to listen on (eg: :8080)")

flag.Parse()

Expand All @@ -86,6 +90,15 @@ func main() {
utils.Config = cfg
logrus.WithField("config", *configPath).WithField("version", version.Version).WithField("chainName", utils.Config.Chain.ClConfig.ConfigName).Printf("starting")

if *metricsAddr != "" {
go func(addr string) {
logrus.Infof("serving metrics on %v", addr)
if err := metrics.Serve(addr); err != nil {
logrus.WithError(err).Fatal("Error serving metrics")
}
}(*metricsAddr)
}

// enable pprof endpoint if requested
if utils.Config.Pprof.Enabled {
go func() {
Expand Down Expand Up @@ -367,7 +380,7 @@ func main() {
}

if *enableEnsUpdater {
err := bt.ImportEnsUpdates(client.GetNativeClient(), 1000)
err := bt.ImportEnsUpdates(client.GetNativeClient(), *ensBatchSize)
if err != nil {
utils.LogError(err, "error importing ens updates", 0, nil)
continue
Expand Down Expand Up @@ -615,12 +628,21 @@ func IndexFromNode(bt *db.Bigtable, client *rpc.ErigonClient, start, end, concur
default:
}

startTime := time.Now()
defer func() {
metrics.TaskDuration.WithLabelValues("bt_index_from_node").Observe(time.Since(startTime).Seconds())
}()

blockStartTs := time.Now()
bc, timings, err := client.GetBlock(i, traceMode)
if err != nil {
return fmt.Errorf("error getting block: %v from ethereum node err: %w", i, err)
}

metrics.TaskDuration.WithLabelValues("rpc_el_get_block_headers").Observe(timings.Headers.Seconds())
metrics.TaskDuration.WithLabelValues("rpc_el_get_block_receipts").Observe(timings.Receipts.Seconds())
metrics.TaskDuration.WithLabelValues("rpc_el_get_block_traces").Observe(timings.Traces.Seconds())

dbStart := time.Now()
err = bt.SaveBlock(bc)
if err != nil {
Expand Down
143 changes: 87 additions & 56 deletions cmd/misc/main.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -61,11 +61,15 @@ var opts = struct {
Key string
ValidatorNameRanges string
Email string
Name string
DryRun bool
Yes bool
}{}

var bt *db.Bigtable
var erigonClient *rpc.ErigonClient
var lighthouseClient *rpc.LighthouseClient
var rpcClient *rpc.LighthouseClient

func main() {
statsPartitionCommand := commands.StatsMigratorCommand{}
Expand All @@ -91,6 +95,7 @@ func main() {
flag.StringVar(&opts.Addresses, "addresses", "", "Comma separated list of addresses that should be processed by the command")
flag.StringVar(&opts.Columns, "columns", "", "Comma separated list of columns that should be affected by the command")
flag.StringVar(&opts.Email, "email", "", "Email of the user")
flag.StringVar(&opts.Name, "name", "", "Name")
flag.BoolVar(&opts.Yes, "yes", false, "Answer yes to all questions")
dryRun := flag.String("dry-run", "true", "if 'false' it deletes all rows starting with the key, per default it only logs the rows that would be deleted, but does not really delete them")
versionFlag := flag.Bool("version", false, "Show version and exit")
Expand All @@ -115,64 +120,90 @@ func main() {
utils.Config = cfg

chainIdString := strconv.FormatUint(utils.Config.Chain.ClConfig.DepositChainID, 10)
chainIDBig := new(big.Int).SetUint64(utils.Config.Chain.ClConfig.DepositChainID)

bt, err := db.InitBigtable(utils.Config.Bigtable.Project, utils.Config.Bigtable.Instance, chainIdString, utils.Config.RedisCacheEndpoint)
if err != nil {
utils.LogFatal(err, "error initializing bigtable", 0)
}
wg := &sync.WaitGroup{}
wg.Add(5)

chainIDBig := new(big.Int).SetUint64(utils.Config.Chain.ClConfig.DepositChainID)
rpcClient, err := rpc.NewLighthouseClient("http://"+cfg.Indexer.Node.Host+":"+cfg.Indexer.Node.Port, chainIDBig)
if err != nil {
utils.LogFatal(err, "lighthouse client error", 0)
}
lighthouseClient = rpcClient
go func() {
defer wg.Done()
var err error
bt, err = db.InitBigtable(utils.Config.Bigtable.Project, utils.Config.Bigtable.Instance, chainIdString, utils.Config.RedisCacheEndpoint)
if err != nil {
utils.LogFatal(err, "error initializing bigtable", 0)
}
}()

go func() {
defer wg.Done()
var err error
rpcClient, err = rpc.NewLighthouseClient("http://"+cfg.Indexer.Node.Host+":"+cfg.Indexer.Node.Port, chainIDBig)
if err != nil {
utils.LogFatal(err, "lighthouse client error", 0)
}
lighthouseClient = rpcClient
}()

go func() {
defer wg.Done()
var err error
erigonClient, err = rpc.NewErigonClient(utils.Config.Eth1ErigonEndpoint)
if err != nil {
logrus.Fatalf("error initializing erigon client: %v", err)
}
}()

go func() {
defer wg.Done()
db.MustInitDB(&types.DatabaseConfig{
Username: cfg.WriterDatabase.Username,
Password: cfg.WriterDatabase.Password,
Name: cfg.WriterDatabase.Name,
Host: cfg.WriterDatabase.Host,
Port: cfg.WriterDatabase.Port,
MaxOpenConns: cfg.WriterDatabase.MaxOpenConns,
MaxIdleConns: cfg.WriterDatabase.MaxIdleConns,
SSL: cfg.WriterDatabase.SSL,
}, &types.DatabaseConfig{
Username: cfg.ReaderDatabase.Username,
Password: cfg.ReaderDatabase.Password,
Name: cfg.ReaderDatabase.Name,
Host: cfg.ReaderDatabase.Host,
Port: cfg.ReaderDatabase.Port,
MaxOpenConns: cfg.ReaderDatabase.MaxOpenConns,
MaxIdleConns: cfg.ReaderDatabase.MaxIdleConns,
SSL: cfg.ReaderDatabase.SSL,
}, "pgx", "postgres")
}()

go func() {
defer wg.Done()
db.MustInitFrontendDB(&types.DatabaseConfig{
Username: cfg.Frontend.WriterDatabase.Username,
Password: cfg.Frontend.WriterDatabase.Password,
Name: cfg.Frontend.WriterDatabase.Name,
Host: cfg.Frontend.WriterDatabase.Host,
Port: cfg.Frontend.WriterDatabase.Port,
MaxOpenConns: cfg.Frontend.WriterDatabase.MaxOpenConns,
MaxIdleConns: cfg.Frontend.WriterDatabase.MaxIdleConns,
SSL: cfg.Frontend.WriterDatabase.SSL,
}, &types.DatabaseConfig{
Username: cfg.Frontend.ReaderDatabase.Username,
Password: cfg.Frontend.ReaderDatabase.Password,
Name: cfg.Frontend.ReaderDatabase.Name,
Host: cfg.Frontend.ReaderDatabase.Host,
Port: cfg.Frontend.ReaderDatabase.Port,
MaxOpenConns: cfg.Frontend.ReaderDatabase.MaxOpenConns,
MaxIdleConns: cfg.Frontend.ReaderDatabase.MaxIdleConns,
SSL: cfg.Frontend.ReaderDatabase.SSL,
}, "pgx", "postgres")
}()

wg.Wait()

erigonClient, err := rpc.NewErigonClient(utils.Config.Eth1ErigonEndpoint)
if err != nil {
logrus.Fatalf("error initializing erigon client: %v", err)
}

db.MustInitDB(&types.DatabaseConfig{
Username: cfg.WriterDatabase.Username,
Password: cfg.WriterDatabase.Password,
Name: cfg.WriterDatabase.Name,
Host: cfg.WriterDatabase.Host,
Port: cfg.WriterDatabase.Port,
MaxOpenConns: cfg.WriterDatabase.MaxOpenConns,
MaxIdleConns: cfg.WriterDatabase.MaxIdleConns,
SSL: cfg.WriterDatabase.SSL,
}, &types.DatabaseConfig{
Username: cfg.ReaderDatabase.Username,
Password: cfg.ReaderDatabase.Password,
Name: cfg.ReaderDatabase.Name,
Host: cfg.ReaderDatabase.Host,
Port: cfg.ReaderDatabase.Port,
MaxOpenConns: cfg.ReaderDatabase.MaxOpenConns,
MaxIdleConns: cfg.ReaderDatabase.MaxIdleConns,
SSL: cfg.ReaderDatabase.SSL,
}, "pgx", "postgres")
defer db.ReaderDb.Close()
defer db.WriterDb.Close()
db.MustInitFrontendDB(&types.DatabaseConfig{
Username: cfg.Frontend.WriterDatabase.Username,
Password: cfg.Frontend.WriterDatabase.Password,
Name: cfg.Frontend.WriterDatabase.Name,
Host: cfg.Frontend.WriterDatabase.Host,
Port: cfg.Frontend.WriterDatabase.Port,
MaxOpenConns: cfg.Frontend.WriterDatabase.MaxOpenConns,
MaxIdleConns: cfg.Frontend.WriterDatabase.MaxIdleConns,
SSL: cfg.Frontend.WriterDatabase.SSL,
}, &types.DatabaseConfig{
Username: cfg.Frontend.ReaderDatabase.Username,
Password: cfg.Frontend.ReaderDatabase.Password,
Name: cfg.Frontend.ReaderDatabase.Name,
Host: cfg.Frontend.ReaderDatabase.Host,
Port: cfg.Frontend.ReaderDatabase.Port,
MaxOpenConns: cfg.Frontend.ReaderDatabase.MaxOpenConns,
MaxIdleConns: cfg.Frontend.ReaderDatabase.MaxIdleConns,
SSL: cfg.Frontend.ReaderDatabase.SSL,
}, "pgx", "postgres")

defer db.FrontendReaderDB.Close()
defer db.FrontendWriterDB.Close()

Expand Down Expand Up @@ -410,7 +441,7 @@ func main() {
case "fix-epochs":
err = fixEpochs()
default:
utils.LogFatal(nil, fmt.Sprintf("unknown command %s", opts.Command), 0)
utils.LogFatal(nil, fmt.Sprintf("unknown command %s", opts.Command), 2)
}

if err != nil {
Expand Down Expand Up @@ -512,7 +543,7 @@ func disableUserPerEmail() error {
}

func fixEns(erigonClient *rpc.ErigonClient) error {
logrus.Infof("command: fix-ens")
logrus.WithField("dry", opts.DryRun).Infof("command: fix-ens")
addrs := []struct {
Address []byte `db:"address"`
EnsName string `db:"ens_name"`
Expand Down Expand Up @@ -574,7 +605,7 @@ func fixEns(erigonClient *rpc.ErigonClient) error {
reverseName, err := go_ens.ReverseResolve(erigonClient.GetNativeClient(), dbAddr)
if err != nil {
if err.Error() == "not a resolver" || err.Error() == "no resolution" {
logrus.WithFields(logrus.Fields{"addr": fmt.Sprintf("%#x", addr.Address), "name": addr.EnsName, "reason": fmt.Sprintf("failed reverse-resolve: %v", err.Error())}).Warnf("updating ens entry: is_primary_name = false")
logrus.WithFields(logrus.Fields{"addr": dbAddr, "name": addr.EnsName, "reason": fmt.Sprintf("failed reverse-resolve: %v", err.Error())}).Warnf("updating ens entry: is_primary_name = false")
if !opts.DryRun {
_, err = db.WriterDb.Exec(`update ens set is_primary_name = false where address = $1 and ens_name = $2`, addr.Address, addr.EnsName)
if err != nil {
Expand Down
81 changes: 81 additions & 0 deletions contracts/ens/ens.go
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,81 @@
package ens

import (
"eth2-exporter/utils"

"github.com/ethereum/go-ethereum/accounts/abi"
"github.com/ethereum/go-ethereum/accounts/abi/bind"
"github.com/ethereum/go-ethereum/common"
)

var ENSCrontractAddressesEthereum = map[string]string{
"0x00000000000C2E074eC69A0dFb2997BA6C7d2e1e": "Registry",
"0x253553366Da8546fC250F225fe3d25d0C782303b": "ETHRegistrarController",
"0x283Af0B28c62C092C9727F1Ee09c02CA627EB7F5": "OldEnsRegistrarController",
}

var ENSCrontractAddressesHolesky = map[string]string{
"0x00000000000C2E074eC69A0dFb2997BA6C7d2e1e": "Registry",
"0x179Be112b24Ad4cFC392eF8924DfA08C20Ad8583": "ETHRegistrarController",
"0x283Af0B28c62C092C9727F1Ee09c02CA627EB7F5": "OldEnsRegistrarController",
}

var ENSCrontractAddressesSepolia = map[string]string{
"0x00000000000C2E074eC69A0dFb2997BA6C7d2e1e": "Registry",
"0xFED6a969AaA60E4961FCD3EBF1A2e8913ac65B72": "ETHRegistrarController",
"0x283Af0B28c62C092C9727F1Ee09c02CA627EB7F5": "OldEnsRegistrarController",
}

var ENSRegistryParsedABI, ENSBaseRegistrarParsedABI, ENSOldRegistrarControllerParsedABI, ENSPublicResolverParsedABI, ENSETHRegistrarControllerParsedABI *abi.ABI

var ENSRegistryContract, ENSBaseRegistrarContract, ENSOldRegistrarControllerContract, ENSPublicResolverContract, ENSETHRegistrarControllerContract *bind.BoundContract

func init() {
var err error

ENSRegistryParsedABI, err = ENSRegistryMetaData.GetAbi()
if err != nil {
utils.LogFatal(err, "error getting ens-registry-abi", 0)
}
ENSRegistryParsedABI, err = ENSRegistryMetaData.GetAbi()
if err != nil {
utils.LogFatal(err, "error getting ens-registry-abi", 0)
}
ENSBaseRegistrarParsedABI, err = ENSBaseRegistrarMetaData.GetAbi()
if err != nil {
utils.LogFatal(err, "error getting ens-base-regsitrar-abi", 0)
}
ENSOldRegistrarControllerParsedABI, err = ENSOldRegistrarControllerMetaData.GetAbi()
if err != nil {
utils.LogFatal(err, "error getting ens-old-registrar-controller-abi", 0)
}
ENSPublicResolverParsedABI, err = ENSPublicResolverMetaData.GetAbi()
if err != nil {
utils.LogFatal(err, "error getting ens-public-resolver-abi", 0)
}
ENSETHRegistrarControllerParsedABI, err = ENSETHRegistrarControllerMetaData.GetAbi()
if err != nil {
utils.LogFatal(err, "error getting ens-eth-registrar-controller-abi", 0)
}

ENSRegistryContract = bind.NewBoundContract(common.Address{}, *ENSRegistryParsedABI, nil, nil, nil)
if err != nil {
utils.LogFatal(err, "error creating ens-registry-contract", 0)
}
ENSBaseRegistrarContract = bind.NewBoundContract(common.Address{}, *ENSBaseRegistrarParsedABI, nil, nil, nil)
if err != nil {
utils.LogFatal(err, "error creating ens-base-registrar-contract", 0)
}
ENSOldRegistrarControllerContract = bind.NewBoundContract(common.Address{}, *ENSOldRegistrarControllerParsedABI, nil, nil, nil)
if err != nil {
utils.LogFatal(err, "error creating ens-old-registrar-controller-contract", 0)
}
ENSPublicResolverContract = bind.NewBoundContract(common.Address{}, *ENSPublicResolverParsedABI, nil, nil, nil)
if err != nil {
utils.LogFatal(err, "error creating ens-public-resolver-contract", 0)
}
ENSETHRegistrarControllerContract = bind.NewBoundContract(common.Address{}, *ENSETHRegistrarControllerParsedABI, nil, nil, nil)
if err != nil {
utils.LogFatal(err, "error creating ens-eth-registrar-controller-contract", 0)
}
}
Loading

0 comments on commit f879104

Please sign in to comment.