From b230a020060f2425478dc3c9b17d4c5dc38c8aa5 Mon Sep 17 00:00:00 2001 From: Fynn Date: Wed, 15 May 2024 15:28:57 +0800 Subject: [PATCH] cmd: fix memory leak when big dataset (#2455) --- cmd/geth/dbcmd.go | 21 ++- trie/inspect_trie.go | 350 ++++++++++++++++++++++--------------------- 2 files changed, 198 insertions(+), 173 deletions(-) diff --git a/cmd/geth/dbcmd.go b/cmd/geth/dbcmd.go index 5a0aaf011a..1cbf031975 100644 --- a/cmd/geth/dbcmd.go +++ b/cmd/geth/dbcmd.go @@ -106,12 +106,12 @@ Remove blockchain and state databases`, dbInspectTrieCmd = &cli.Command{ Action: inspectTrie, Name: "inspect-trie", - ArgsUsage: " ", + ArgsUsage: " ", Flags: []cli.Flag{ utils.DataDirFlag, utils.SyncModeFlag, }, - Usage: "Inspect the MPT tree of the account and contract.", + Usage: "Inspect the MPT tree of the account and contract. 'blocknum' can be latest/snapshot/number. 'topn' means output the top N storage tries info ranked by the total number of TrieNodes", Description: `This commands iterates the entrie WorldState.`, } dbCheckStateContentCmd = &cli.Command{ @@ -386,6 +386,7 @@ func inspectTrie(ctx *cli.Context) error { blockNumber uint64 trieRootHash common.Hash jobnum uint64 + topN uint64 ) stack, _ := makeConfigNode(ctx) @@ -411,12 +412,25 @@ func inspectTrie(ctx *cli.Context) error { if ctx.NArg() == 1 { jobnum = 1000 + topN = 10 + } else if ctx.NArg() == 2 { + var err error + jobnum, err = strconv.ParseUint(ctx.Args().Get(1), 10, 64) + if err != nil { + return fmt.Errorf("failed to Parse jobnum, Args[1]: %v, err: %v", ctx.Args().Get(1), err) + } + topN = 10 } else { var err error jobnum, err = strconv.ParseUint(ctx.Args().Get(1), 10, 64) if err != nil { return fmt.Errorf("failed to Parse jobnum, Args[1]: %v, err: %v", ctx.Args().Get(1), err) } + + topN, err = strconv.ParseUint(ctx.Args().Get(2), 10, 64) + if err != nil { + return fmt.Errorf("failed to Parse topn, Args[1]: %v, err: %v", ctx.Args().Get(1), err) + } } if blockNumber != math.MaxUint64 { @@ -437,6 +451,7 @@ func inspectTrie(ctx *cli.Context) error { if dbScheme == rawdb.PathScheme { config = &triedb.Config{ PathDB: utils.PathDBConfigAddJournalFilePath(stack, pathdb.ReadOnly), + Cache: 0, } } else if dbScheme == rawdb.HashScheme { config = triedb.HashDefaults @@ -448,7 +463,7 @@ func inspectTrie(ctx *cli.Context) error { fmt.Printf("fail to new trie tree, err: %v, rootHash: %v\n", err, trieRootHash.String()) return err } - theInspect, err := trie.NewInspector(theTrie, triedb, trieRootHash, blockNumber, jobnum) + theInspect, err := trie.NewInspector(theTrie, triedb, trieRootHash, blockNumber, jobnum, int(topN)) if err != nil { return err } diff --git a/trie/inspect_trie.go b/trie/inspect_trie.go index 885d3c7454..89f691a36d 100644 --- a/trie/inspect_trie.go +++ b/trie/inspect_trie.go @@ -4,17 +4,15 @@ import ( "bytes" "errors" "fmt" - "math/big" - "os" "runtime" - "sort" - "strconv" + "strings" "sync" "sync/atomic" "time" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/common/mclock" "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/rlp" @@ -26,63 +24,113 @@ import ( "golang.org/x/sync/semaphore" ) -type Account struct { - Nonce uint64 - Balance *big.Int - Root common.Hash // merkle root of the storage trie - CodeHash []byte -} - type Database interface { database.Database Scheme() string Cap(limit common.StorageSize) error DiskDB() ethdb.Database } + +const TopN = 3 + type Inspector struct { trie *Trie // traverse trie db Database stateRootHash common.Hash - blocknum uint64 + blockNum uint64 root node // root of triedb - totalNum uint64 - wg sync.WaitGroup - statLock sync.RWMutex - result map[string]*TrieTreeStat sem *semaphore.Weighted eoaAccountNums uint64 + + wg sync.WaitGroup + + results stat + topN int + + totalAccountNum atomic.Uint64 + totalStorageNum atomic.Uint64 + lastTime mclock.AbsTime +} + +type stat struct { + lock sync.RWMutex + account *trieStat + storageTopN []*trieStat + storageTopNTotal []uint64 + storageTotal nodeStat + storageTrieNum uint64 } -type TrieTreeStat struct { - is_account_trie bool - theNodeStatByLevel [15]NodeStat - totalNodeStat NodeStat +type trieStat struct { + owner common.Hash + totalNodeStat nodeStat + nodeStatByLevel [16]nodeStat } -type NodeStat struct { - ShortNodeCnt uint64 - FullNodeCnt uint64 - ValueNodeCnt uint64 +type nodeStat struct { + ShortNodeCnt atomic.Uint64 + FullNodeCnt atomic.Uint64 + ValueNodeCnt atomic.Uint64 +} + +func (ns *nodeStat) IsEmpty() bool { + if ns.FullNodeCnt.Load() == 0 && ns.ShortNodeCnt.Load() == 0 && ns.ValueNodeCnt.Load() == 0 { + return true + } + return false +} + +func (s *stat) add(ts *trieStat, topN int) { + s.lock.Lock() + defer s.lock.Unlock() + if ts.owner == (common.Hash{}) { + s.account = ts + return + } + + total := ts.totalNodeStat.ValueNodeCnt.Load() + ts.totalNodeStat.FullNodeCnt.Load() + ts.totalNodeStat.ShortNodeCnt.Load() + if len(s.storageTopNTotal) == 0 || total > s.storageTopNTotal[len(s.storageTopNTotal)-1] { + var ( + i int + t uint64 + ) + for i, t = range s.storageTopNTotal { + if total < t { + continue + } + break + } + s.storageTopNTotal = append(s.storageTopNTotal[:i], append([]uint64{total}, s.storageTopNTotal[i:]...)...) + s.storageTopN = append(s.storageTopN[:i], append([]*trieStat{ts}, s.storageTopN[i:]...)...) + if len(s.storageTopN) > topN { + s.storageTopNTotal = s.storageTopNTotal[:topN] + s.storageTopN = s.storageTopN[:topN] + } + } + + s.storageTotal.ShortNodeCnt.Add(ts.totalNodeStat.ShortNodeCnt.Load()) + s.storageTotal.ValueNodeCnt.Add(ts.totalNodeStat.ValueNodeCnt.Load()) + s.storageTotal.FullNodeCnt.Add(ts.totalNodeStat.FullNodeCnt.Load()) + s.storageTrieNum++ } -func (trieStat *TrieTreeStat) AtomicAdd(theNode node, height uint32) { +func (trieStat *trieStat) add(theNode node, height int) { switch (theNode).(type) { case *shortNode: - atomic.AddUint64(&trieStat.totalNodeStat.ShortNodeCnt, 1) - atomic.AddUint64(&(trieStat.theNodeStatByLevel[height].ShortNodeCnt), 1) + trieStat.totalNodeStat.ShortNodeCnt.Add(1) + trieStat.nodeStatByLevel[height].ShortNodeCnt.Add(1) case *fullNode: - atomic.AddUint64(&trieStat.totalNodeStat.FullNodeCnt, 1) - atomic.AddUint64(&trieStat.theNodeStatByLevel[height].FullNodeCnt, 1) + trieStat.totalNodeStat.FullNodeCnt.Add(1) + trieStat.nodeStatByLevel[height].FullNodeCnt.Add(1) case valueNode: - atomic.AddUint64(&trieStat.totalNodeStat.ValueNodeCnt, 1) - atomic.AddUint64(&((trieStat.theNodeStatByLevel[height]).ValueNodeCnt), 1) - default: - panic(errors.New("Invalid node type to statistics")) + trieStat.totalNodeStat.ValueNodeCnt.Add(1) + trieStat.nodeStatByLevel[height].ValueNodeCnt.Add(1) } } -func (trieStat *TrieTreeStat) Display(ownerAddress string, treeType string) { - table := tablewriter.NewWriter(os.Stdout) +func (trieStat *trieStat) Display(ownerAddress string, treeType string) string { + sw := new(strings.Builder) + table := tablewriter.NewWriter(sw) table.SetHeader([]string{"-", "Level", "ShortNodeCnt", "FullNodeCnt", "ValueNodeCnt"}) if ownerAddress == "" { table.SetCaption(true, fmt.Sprintf("%v", treeType)) @@ -90,38 +138,27 @@ func (trieStat *TrieTreeStat) Display(ownerAddress string, treeType string) { table.SetCaption(true, fmt.Sprintf("%v-%v", treeType, ownerAddress)) } table.SetAlignment(1) - for i := 0; i < len(trieStat.theNodeStatByLevel); i++ { - nodeStat := trieStat.theNodeStatByLevel[i] - if nodeStat.FullNodeCnt == 0 && nodeStat.ShortNodeCnt == 0 && nodeStat.ValueNodeCnt == 0 { - break + + for i := range trieStat.nodeStatByLevel { + if trieStat.nodeStatByLevel[i].IsEmpty() { + continue } table.AppendBulk([][]string{ - {"-", strconv.Itoa(i), nodeStat.ShortNodeCount(), nodeStat.FullNodeCount(), nodeStat.ValueNodeCount()}, + {"-", fmt.Sprintf("%d", i), + fmt.Sprintf("%d", trieStat.nodeStatByLevel[i].ShortNodeCnt.Load()), + fmt.Sprintf("%d", trieStat.nodeStatByLevel[i].FullNodeCnt.Load()), + fmt.Sprintf("%d", trieStat.nodeStatByLevel[i].ValueNodeCnt.Load())}, }) } table.AppendBulk([][]string{ - {"Total", "-", trieStat.totalNodeStat.ShortNodeCount(), trieStat.totalNodeStat.FullNodeCount(), trieStat.totalNodeStat.ValueNodeCount()}, + {"Total", "-", fmt.Sprintf("%d", trieStat.totalNodeStat.ShortNodeCnt.Load()), fmt.Sprintf("%d", trieStat.totalNodeStat.FullNodeCnt.Load()), fmt.Sprintf("%d", trieStat.totalNodeStat.ValueNodeCnt.Load())}, }) table.Render() -} - -func Uint64ToString(cnt uint64) string { - return fmt.Sprintf("%v", cnt) -} - -func (nodeStat *NodeStat) ShortNodeCount() string { - return Uint64ToString(nodeStat.ShortNodeCnt) -} - -func (nodeStat *NodeStat) FullNodeCount() string { - return Uint64ToString(nodeStat.FullNodeCnt) -} -func (nodeStat *NodeStat) ValueNodeCount() string { - return Uint64ToString(nodeStat.ValueNodeCnt) + return sw.String() } // NewInspector return a inspector obj -func NewInspector(tr *Trie, db Database, stateRootHash common.Hash, blocknum uint64, jobnum uint64) (*Inspector, error) { +func NewInspector(tr *Trie, db Database, stateRootHash common.Hash, blockNum uint64, jobNum uint64, topN int) (*Inspector, error) { if tr == nil { return nil, errors.New("trie is nil") } @@ -131,15 +168,20 @@ func NewInspector(tr *Trie, db Database, stateRootHash common.Hash, blocknum uin } ins := &Inspector{ - trie: tr, - db: db, - stateRootHash: stateRootHash, - blocknum: blocknum, - root: tr.root, - result: make(map[string]*TrieTreeStat), - totalNum: (uint64)(0), - wg: sync.WaitGroup{}, - sem: semaphore.NewWeighted(int64(jobnum)), + trie: tr, + db: db, + stateRootHash: stateRootHash, + blockNum: blockNum, + root: tr.root, + results: stat{}, + topN: topN, + totalAccountNum: atomic.Uint64{}, + totalStorageNum: atomic.Uint64{}, + lastTime: mclock.Now(), + sem: semaphore.NewWeighted(int64(jobNum)), + + wg: sync.WaitGroup{}, + eoaAccountNums: 0, } @@ -147,155 +189,123 @@ func NewInspector(tr *Trie, db Database, stateRootHash common.Hash, blocknum uin } // Run statistics, external call -func (inspect *Inspector) Run() { - accountTrieStat := &TrieTreeStat{ - is_account_trie: true, - } - if inspect.db.Scheme() == rawdb.HashScheme { - ticker := time.NewTicker(30 * time.Second) - go func() { - defer ticker.Stop() - for range ticker.C { - inspect.db.Cap(DEFAULT_TRIEDBCACHE_SIZE) +func (s *Inspector) Run() { + ticker := time.NewTicker(30 * time.Second) + go func() { + defer ticker.Stop() + for range ticker.C { + if s.db.Scheme() == rawdb.HashScheme { + s.db.Cap(DEFAULT_TRIEDBCACHE_SIZE) } - }() - } - - if _, ok := inspect.result[""]; !ok { - inspect.result[""] = accountTrieStat - } - log.Info("Find Account Trie Tree", "rootHash: ", inspect.trie.Hash().String(), "BlockNum: ", inspect.blocknum) - - inspect.ConcurrentTraversal(inspect.trie, accountTrieStat, inspect.root, 0, []byte{}) - inspect.wg.Wait() -} + runtime.GC() + } + }() -func (inspect *Inspector) SubConcurrentTraversal(theTrie *Trie, theTrieTreeStat *TrieTreeStat, theNode node, height uint32, path []byte) { - inspect.ConcurrentTraversal(theTrie, theTrieTreeStat, theNode, height, path) - inspect.wg.Done() -} + log.Info("Find Account Trie Tree", "rootHash: ", s.trie.Hash().String(), "BlockNum: ", s.blockNum) -func (inspect *Inspector) ConcurrentTraversal(theTrie *Trie, theTrieTreeStat *TrieTreeStat, theNode node, height uint32, path []byte) { - // print process progress - total_num := atomic.AddUint64(&inspect.totalNum, 1) - if total_num%100000 == 0 { - fmt.Printf("Complete progress: %v, go routines Num: %v\n", total_num, runtime.NumGoroutine()) + ts := &trieStat{ + owner: common.Hash{}, } + s.traversal(s.trie, ts, s.root, 0, []byte{}) + s.results.add(ts, s.topN) + s.wg.Wait() +} +func (s *Inspector) traversal(trie *Trie, ts *trieStat, n node, height int, path []byte) { // nil node - if theNode == nil { + if n == nil { return } - switch current := (theNode).(type) { + ts.add(n, height) + + switch current := (n).(type) { case *shortNode: - inspect.ConcurrentTraversal(theTrie, theTrieTreeStat, current.Val, height, append(path, current.Key...)) + s.traversal(trie, ts, current.Val, height, append(path, current.Key...)) case *fullNode: for idx, child := range current.Children { if child == nil { continue } - childPath := append(path, byte(idx)) - if inspect.sem.TryAcquire(1) { - inspect.wg.Add(1) - dst := make([]byte, len(childPath)) - copy(dst, childPath) - go inspect.SubConcurrentTraversal(theTrie, theTrieTreeStat, child, height+1, dst) - } else { - inspect.ConcurrentTraversal(theTrie, theTrieTreeStat, child, height+1, childPath) - } + p := common.CopyBytes(append(path, byte(idx))) + s.traversal(trie, ts, child, height+1, p) } case hashNode: - n, err := theTrie.resloveWithoutTrack(current, path) + tn, err := trie.resloveWithoutTrack(current, path) if err != nil { - fmt.Printf("Resolve HashNode error: %v, TrieRoot: %v, Height: %v, Path: %v\n", err, theTrie.Hash().String(), height+1, path) + fmt.Printf("Resolve HashNode error: %v, TrieRoot: %v, Height: %v, Path: %v\n", err, trie.Hash().String(), height+1, path) return } - inspect.ConcurrentTraversal(theTrie, theTrieTreeStat, n, height, path) - return + s.PrintProgress(trie) + s.traversal(trie, ts, tn, height, path) case valueNode: if !hasTerm(path) { break } - var account Account + var account types.StateAccount if err := rlp.Decode(bytes.NewReader(current), &account); err != nil { break } if common.BytesToHash(account.CodeHash) == types.EmptyCodeHash { - inspect.eoaAccountNums++ + s.eoaAccountNums++ } if account.Root == (common.Hash{}) || account.Root == types.EmptyRootHash { break } ownerAddress := common.BytesToHash(hexToCompact(path)) - contractTrie, err := New(StorageTrieID(inspect.stateRootHash, ownerAddress, account.Root), inspect.db) + contractTrie, err := New(StorageTrieID(s.stateRootHash, ownerAddress, account.Root), s.db) if err != nil { - fmt.Printf("New contract trie node: %v, error: %v, Height: %v, Path: %v\n", theNode, err, height, path) - break + panic(err) } contractTrie.tracer.reset() - trieStat := &TrieTreeStat{ - is_account_trie: false, - } - inspect.statLock.Lock() - if _, ok := inspect.result[ownerAddress.String()]; !ok { - inspect.result[ownerAddress.String()] = trieStat + if s.sem.TryAcquire(1) { + s.wg.Add(1) + go func() { + t := &trieStat{ + owner: ownerAddress, + } + s.traversal(contractTrie, t, contractTrie.root, 0, []byte{}) + s.results.add(t, s.topN) + s.sem.Release(1) + s.wg.Done() + }() + } else { + t := &trieStat{ + owner: ownerAddress, + } + s.traversal(contractTrie, t, contractTrie.root, 0, []byte{}) + s.results.add(t, s.topN) } - inspect.statLock.Unlock() - - // log.Info("Find Contract Trie Tree, rootHash: ", contractTrie.Hash().String(), "") - inspect.wg.Add(1) - go inspect.SubConcurrentTraversal(contractTrie, trieStat, contractTrie.root, 0, []byte{}) default: - panic(errors.New("Invalid node type to traverse.")) + panic(errors.New("invalid node type to traverse")) } - theTrieTreeStat.AtomicAdd(theNode, height) } -func (inspect *Inspector) DisplayResult() { - // display root hash - if _, ok := inspect.result[""]; !ok { - log.Info("Display result error", "missing account trie") - return +func (s *Inspector) PrintProgress(t *Trie) { + var ( + elapsed = mclock.Now().Sub(s.lastTime) + ) + if t.owner == (common.Hash{}) { + s.totalAccountNum.Add(1) + } else { + s.totalStorageNum.Add(1) } - inspect.result[""].Display("", "AccountTrie") - - type SortedTrie struct { - totalNum uint64 - ownerAddress string + if elapsed > 4*time.Second { + log.Info("traversal progress", "TotalAccountNum", s.totalAccountNum.Load(), "TotalStorageNum", s.totalStorageNum.Load(), "Goroutine", runtime.NumGoroutine()) + s.lastTime = mclock.Now() } - // display contract trie - var sortedTriesByNums []SortedTrie - var totalContactsNodeStat NodeStat - var contractTrieCnt uint64 = 0 +} - for ownerAddress, stat := range inspect.result { - if ownerAddress == "" { - continue - } - contractTrieCnt++ - totalContactsNodeStat.ShortNodeCnt += stat.totalNodeStat.ShortNodeCnt - totalContactsNodeStat.FullNodeCnt += stat.totalNodeStat.FullNodeCnt - totalContactsNodeStat.ValueNodeCnt += stat.totalNodeStat.ValueNodeCnt - totalNodeCnt := stat.totalNodeStat.ShortNodeCnt + stat.totalNodeStat.ValueNodeCnt + stat.totalNodeStat.FullNodeCnt - sortedTriesByNums = append(sortedTriesByNums, SortedTrie{totalNum: totalNodeCnt, ownerAddress: ownerAddress}) - } - sort.Slice(sortedTriesByNums, func(i, j int) bool { - return sortedTriesByNums[i].totalNum > sortedTriesByNums[j].totalNum - }) - fmt.Println("EOA accounts num: ", inspect.eoaAccountNums) - // only display top 5 - for i, t := range sortedTriesByNums { - if i > 5 { - break - } - if stat, ok := inspect.result[t.ownerAddress]; !ok { - log.Error("Storage trie stat not found", "ownerAddress", t.ownerAddress) - } else { - stat.Display(t.ownerAddress, "ContractTrie") - } +func (s *Inspector) DisplayResult() { + // display root hash + fmt.Println(s.results.account.Display("", "AccountTrie")) + fmt.Println("EOA accounts num: ", s.eoaAccountNums) + + // display contract trie + for _, st := range s.results.storageTopN { + fmt.Println(st.Display(st.owner.String(), "StorageTrie")) } fmt.Printf("Contract Trie, total trie num: %v, ShortNodeCnt: %v, FullNodeCnt: %v, ValueNodeCnt: %v\n", - contractTrieCnt, totalContactsNodeStat.ShortNodeCnt, totalContactsNodeStat.FullNodeCnt, totalContactsNodeStat.ValueNodeCnt) + s.results.storageTrieNum, s.results.storageTotal.ShortNodeCnt.Load(), s.results.storageTotal.FullNodeCnt.Load(), s.results.storageTotal.ValueNodeCnt.Load()) }