Skip to content

Commit

Permalink
(BIDS-2550) fixed contract invocation detection by adding metadata co…
Browse files Browse the repository at this point in the history 
…lumn (#2618)

* (BIDS-2550) saving contract updates to bigtable

(BIDS-2550) use field for blob transactions

(BIDS-2550) save contract updates to bigtable

(BIDS-2550) removed metadataUpdates indirection; added destruction detection

(BIDS-2550) support for internal transactions
(BIDS-2550) handle contract destruction, show correct data

(BIDS-2550) detect failed transactions
(BIDS-2550) applied (most) suggestions

(BIDS-2550) remove contract state updates on forks

this approach might be a bit naive though, as bigtable probably can't filter by time efficiently
(BIDS-2550) applied feedback

(BIDS-2550) fixed invalid bt blockKey timestamps

(BIDS-2550) added bigtable transform
(BIDS-2550) correct variable naming

(BIDS-2550) added new transform to list of all

(BIDS-2550) fixed ts search

(BIDS-2550) fixed failed itx detection, fixed update sorting, variable naming

(BIDS-2550) fixed bigtable rowrange

(BIDS-2550) using distinct contract update bt key

(BIDS-2550) add column filter to ClearByPrefix misc command
(BIDS-2550) delete keys output clarified

(BIDS-2550) ui issue fixed, bit assumptions adjusted
(BIDS-2550) fixed columns filter

(BIDS-2550) disabled column filter

* (BIDS-2550) fixed column filter

* (BIDS-2550) added contract detection for parity traces

* (BIDS-2550) fix parity contract detection

* (BIDS-2550) using utils.LogFatal
  • Loading branch information
@remoterami
remoterami authored Feb 6, 2024
1 parent a0ad636 commit ed188e4
Show file tree
Hide file tree
Showing 13 changed files with 1,026 additions and 654 deletions.
186 changes: 10 additions & 176 deletions cmd/eth1indexer/main.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -163,44 +163,10 @@ func main() {
}
}()
}
// err = UpdateTokenPrices(bt, client, "tokenlists/tokens.uniswap.org.json")
// if err != nil {
// logrus.Fatal(err)
// }
// return

if *enableFullBalanceUpdater {
ProcessMetadataUpdates(bt, client, balanceUpdaterPrefix, *balanceUpdaterBatchSize, -1)
return
// currentKey := balanceUpdaterPrefix // "1:00028ebf7d36c5779c1deddf3ba72761fd46c8aa"
// for {
// keys, pairs, err := bt.GetMetadata(currentKey, *balanceUpdaterBatchSize)
// if err != nil {
// logrus.Fatal(err)
// }

// if len(keys) == 0 {
// logrus.Infof("done")
// return
// }
// // for _, pair := range pairs {
// // logrus.Info(pair)
// // }

// logrus.Infof("currently at %v, processing balances for %v pairs", currentKey, len(pairs))
// balances, err := client.GetBalances(pairs, 1, 4)
// if err != nil {
// logrus.Fatal(err)
// }
// // for _, balance := range balances {
// // logrus.Infof("%x %x %s", balance.Address, balance.Token, new(big.Int).SetBytes(balance.Balance))
// // }

// err = bt.SaveBalances(balances, []string{})
// if err != nil {
// logrus.Fatal(err)
// }
// currentKey = keys[len(keys)-1]
// }
}

transforms := make([]func(blk *types.Eth1Block, cache *freecache.Cache) (*types.BulkMutations, *types.BulkMutations, error), 0)
Expand All @@ -214,7 +180,8 @@ func main() {
bt.TransformERC1155,
bt.TransformUncle,
bt.TransformWithdrawals,
bt.TransformEnsNameRegistered)
bt.TransformEnsNameRegistered,
bt.TransformContract)

cache := freecache.NewCache(100 * 1024 * 1024) // 100 MB limit

Expand Down Expand Up @@ -297,9 +264,9 @@ func main() {
continueAfterError := false
if lastBlockFromNode > 0 {
if lastBlockFromBlocksTable < int(lastBlockFromNode) {
logrus.Infof("missing blocks %v to %v in blocks table, indexing ...", lastBlockFromBlocksTable, lastBlockFromNode)
logrus.Infof("missing blocks %v to %v in blocks table, indexing ...", lastBlockFromBlocksTable+1, lastBlockFromNode)

startBlock := int64(lastBlockFromBlocksTable) - *offsetBlocks
startBlock := int64(lastBlockFromBlocksTable+1) - *offsetBlocks
if startBlock < 0 {
startBlock = 0
}
Expand Down Expand Up @@ -340,9 +307,9 @@ func main() {
}

if lastBlockFromDataTable < int(lastBlockFromNode) {
logrus.Infof("missing blocks %v to %v in data table, indexing ...", lastBlockFromDataTable, lastBlockFromNode)
logrus.Infof("missing blocks %v to %v in data table, indexing ...", lastBlockFromDataTable+1, lastBlockFromNode)

startBlock := int64(lastBlockFromDataTable) - *offsetData
startBlock := int64(lastBlockFromDataTable+1) - *offsetData
if startBlock < 0 {
startBlock = 0
}
Expand Down Expand Up @@ -498,6 +465,9 @@ func HandleChainReorgs(bt *db.Bigtable, client *rpc.ErigonClient, depth int) err
latestNodeBlockNumber := latestNodeBlock.NumberU64()

// for each block check if block node hash and block db hash match
if depth > int(latestNodeBlockNumber) {
depth = int(latestNodeBlockNumber)
}
for i := latestNodeBlockNumber - uint64(depth); i <= latestNodeBlockNumber; i++ {
nodeBlock, err := client.GetNativeClient().HeaderByNumber(ctx, big.NewInt(int64(i)))
if err != nil {
Expand Down Expand Up @@ -554,80 +524,6 @@ func HandleChainReorgs(bt *db.Bigtable, client *rpc.ErigonClient, depth int) err

func ProcessMetadataUpdates(bt *db.Bigtable, client *rpc.ErigonClient, prefix string, batchSize int, iterations int) {
lastKey := prefix
// for {
// updates, err := bt.GetMetadataUpdates(lastKey, batchSize)
// if err != nil {
// logrus.Fatal(err)
// }

// currentAddress := ""
// tokens := make([]string, 0, 100)
// pairs := make([]string, 0, batchSize)
// for _, update := range updates {
// s := strings.Split(update, ":")

// if len(s) != 3 {
// logrus.Fatalf("%v has an invalid format", update)
// }

// if s[0] != "B" {
// logrus.Fatalf("%v has invalid balance update prefix", update)
// }

// address := s[1]
// token := s[2]
// pairs = append(pairs, update)

// if currentAddress == "" {
// currentAddress = address
// } else if address != currentAddress {
// logrus.Infof("retrieving %v token balances for address %v", len(tokens), currentAddress)
// start := time.Now()
// balances, err := client.GetBalancesForAddresse(currentAddress, tokens)

// if err != nil {
// logrus.Errorf("error during balance checker contract call: %v", err)
// logrus.Infof("retrieving balances via batch rpc calls")
// balances, err = client.GetBalances(pairs)
// if err != nil {
// logrus.Fatal(err)
// }
// }

// logrus.Infof("retrieved %v balances in %v", len(balances), time.Since(start))
// // for i, t := range tokens {
// // if len(balances[i]) > 0 {
// // logrus.Infof("balance of address %v of token %v is %x", currentAddress, t, balances[i])
// // }
// // }
// currentAddress = address
// tokens = make([]string, 0, 100)
// pairs = make([]string, 0, 1000)
// }

// tokens = append(tokens, token)
// }
// logrus.Infof("retrieving %v token balances for address %v", len(tokens), currentAddress)
// start := time.Now()
// balances, err := client.GetBalancesForAddresse(currentAddress, tokens)

// if err != nil {
// logrus.Errorf("error during balance checker contract call: %v", err)
// logrus.Infof("retrieving balances via batch rpc calls")
// balances, err = client.GetBalances(pairs)
// if err != nil {
// logrus.Fatal(err)
// }
// }

// logrus.Infof("retrieved %v balances in %v", len(balances), time.Since(start))
// // for i, t := range tokens {
// // if len(balances[i]) > 0 {
// // logrus.Infof("balance of address %v of token %v is %x", currentAddress, t, balances[i])
// // }
// // }
// lastKey = updates[len(updates)-1]
// }

its := 0
for {
Expand All @@ -642,10 +538,6 @@ func ProcessMetadataUpdates(bt *db.Bigtable, client *rpc.ErigonClient, prefix st
return
}

// for _, b := range balances {
// logrus.Infof("retrieved balance %x for token %x of address %x", b.Balance, b.Token, b.Address)
// }

balances := make([]*types.Eth1AddressBalance, 0, len(pairs))
for b := 0; b < len(pairs); b += batchSize {
start := b
Expand All @@ -670,12 +562,6 @@ func ProcessMetadataUpdates(bt *db.Bigtable, client *rpc.ErigonClient, prefix st
logrus.Errorf("error saving balances to bigtable: %v", err)
return
}
// for i, b := range balances {

// if len(b) > 0 {
// logrus.Infof("balance for key %v is %x", updates[i], b)
// }
// }

lastKey = keys[len(keys)-1]
logrus.Infof("retrieved %v balances in %v, currently at %v", len(balances), time.Since(start), lastKey)
Expand All @@ -686,58 +572,6 @@ func ProcessMetadataUpdates(bt *db.Bigtable, client *rpc.ErigonClient, prefix st
return
}
}
// g := new(errgroup.Group)
// g.SetLimit(batchSize)

// for _, update := range updates {
// update := update

// g.Go(func() error {
// // logrus.Infof("updating balance of key %v", update)
// s := strings.Split(update, ":")

// if len(s) != 3 {
// logrus.Fatalf("%v has an invalid format", update)
// }

// if s[0] != "B" {
// logrus.Fatalf("%v has invalid balance update prefix", update)
// }

// address := s[1]
// token := s[2]

// if token == "00" {
// balance, err := client.GetNativeBalance(address)
// if err != nil {
// logrus.Fatal(err)
// }

// balanceInt := new(big.Int).SetBytes(balance)

// if balanceInt.Cmp(big.NewInt(0)) != 0 {
// logrus.Infof("native balance of %v is %x", address, balanceInt.String())
// }
// } else {
// balance, err := client.GetERC20TokenBalance(address, token)
// if err != nil {
// logrus.Fatal(err)
// }

// balanceInt := new(big.Int).SetBytes(balance)
// if balanceInt.Cmp(big.NewInt(0)) != 0 {
// logrus.Infof("token %v balance of %v is %v", token, address, balanceInt.String())
// }
// }
// return nil
// })
// }

// err = g.Wait()

// if err != nil {
// logrus.Fatal(err)
// }
}

func IndexFromNode(bt *db.Bigtable, client *rpc.ErigonClient, start, end, concurrency int64, traceMode string) error {
Expand Down
12 changes: 7 additions & 5 deletions cmd/misc/main.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -264,7 +264,7 @@ func main() {
case "debug-blocks":
err = debugBlocks()
case "clear-bigtable":
clearBigtable(opts.Table, opts.Family, opts.Key, opts.DryRun, bt)
clearBigtable(opts.Table, opts.Family, opts.Columns, opts.Key, opts.DryRun, bt)
case "index-old-eth1-blocks":
indexOldEth1Blocks(opts.StartBlock, opts.EndBlock, opts.BatchSize, opts.DataConcurrency, opts.Transformers, bt, erigonClient)
case "update-aggregation-bits":
Expand Down Expand Up @@ -1318,10 +1318,10 @@ func compareRewards(dayStart uint64, dayEnd uint64, validator uint64, bt *db.Big

}

func clearBigtable(table string, family string, key string, dryRun bool, bt *db.Bigtable) {
func clearBigtable(table string, family string, columns string, key string, dryRun bool, bt *db.Bigtable) {

if !dryRun {
confirmation := utils.CmdPrompt(fmt.Sprintf("Are you sure you want to delete all big table entries starting with [%v] for family [%v]?", key, family))
confirmation := utils.CmdPrompt(fmt.Sprintf("Are you sure you want to delete all big table entries starting with [%v] for family [%v] and columns [%v]?", key, family, columns))
if confirmation != "yes" {
logrus.Infof("Abort!")
return
Expand All @@ -1341,7 +1341,7 @@ func clearBigtable(table string, family string, key string, dryRun bool, bt *db.
// if err != nil {
// logrus.Fatal(err)
// }
err := bt.ClearByPrefix(table, family, key, dryRun)
err := bt.ClearByPrefix(table, family, columns, key, dryRun)

if err != nil {
logrus.Fatalf("error deleting from bigtable: %v", err)
Expand Down Expand Up @@ -1445,7 +1445,7 @@ func indexOldEth1Blocks(startBlock uint64, endBlock uint64, batchSize uint64, co
logrus.Infof("transformerFlag: %v", transformerFlag)
transformerList := strings.Split(transformerFlag, ",")
if transformerFlag == "all" {
transformerList = []string{"TransformBlock", "TransformTx", "TransformBlobTx", "TransformItx", "TransformERC20", "TransformERC721", "TransformERC1155", "TransformWithdrawals", "TransformUncle", "TransformEnsNameRegistered"}
transformerList = []string{"TransformBlock", "TransformTx", "TransformBlobTx", "TransformItx", "TransformERC20", "TransformERC721", "TransformERC1155", "TransformWithdrawals", "TransformUncle", "TransformEnsNameRegistered", "TransformContract"}
} else if len(transformerList) == 0 {
utils.LogError(nil, "no transformer functions provided", 0)
return
Expand Down Expand Up @@ -1478,6 +1478,8 @@ func indexOldEth1Blocks(startBlock uint64, endBlock uint64, batchSize uint64, co
case "TransformEnsNameRegistered":
transforms = append(transforms, bt.TransformEnsNameRegistered)
importENSChanges = true
case "TransformContract":
transforms = append(transforms, bt.TransformContract)
default:
utils.LogError(nil, "Invalid transformer flag %v", 0)
return
Expand Down
Loading

0 comments on commit ed188e4

Please sign in to comment.