cmd/{geth, utils}: add a command to clear verkle costs (#326)

* cmd/{geth, utils}: add a command to clear verkle costs

fix:  boolean issue

fix: load finalization state in FinalizeAndAssemble (#340)

* Conversion and TransitionTrie fixes (#346)

* fixes

Signed-off-by: Ignacio Hagopian <[email protected]>

* remove old comment

Signed-off-by: Ignacio Hagopian <[email protected]>

---------

Signed-off-by: Ignacio Hagopian <[email protected]>

* trace cleanup

---------

Signed-off-by: Ignacio Hagopian <[email protected]>
Co-authored-by: Ignacio Hagopian <[email protected]>

cmd/geth/config.go

@@ -183,6 +183,9 @@ func makeFullNode(ctx *cli.Context) (*node.Node, ethapi.Backend) {
 		v := ctx.Uint64(utils.OverrideOverlayStride.Name)
 		cfg.Eth.OverrideOverlayStride = &v
 	}
+	if ctx.IsSet(utils.ClearVerkleCosts.Name) {
+		params.ClearVerkleWitnessCosts()
+	}
 	backend, eth := utils.RegisterEthService(stack, &cfg.Eth)
 
 	// Configure log filter RPC API.

cmd/geth/main.go

@@ -71,6 +71,7 @@ var (
 		utils.OverrideCancun,
 		utils.OverridePrague,
 		utils.OverrideProofInBlock,
+		utils.ClearVerkleCosts,
 		utils.EnablePersonal,
 		utils.TxPoolLocalsFlag,
 		utils.TxPoolNoLocalsFlag,

cmd/utils/flags.go

@@ -284,6 +284,11 @@ var (
 		Usage:    "Manually specify the proof-in-block setting",
 		Category: flags.EthCategory,
 	}
+	ClearVerkleCosts = &cli.BoolFlag{
+		Name:     "clear.verkle.costs",
+		Usage:    "Clear verkle costs (for shadow forks)",
+		Category: flags.EthCategory,
+	}
 	// Light server and client settings
 	LightServeFlag = &cli.IntFlag{
 		Name:     "light.serve",

consensus/beacon/consensus.go

@@ -28,6 +28,7 @@ import (
 	"github.com/ethereum/go-ethereum/core/overlay"
 	"github.com/ethereum/go-ethereum/core/state"
 	"github.com/ethereum/go-ethereum/core/types"
+	"github.com/ethereum/go-ethereum/log"
 	"github.com/ethereum/go-ethereum/params"
 	"github.com/ethereum/go-ethereum/rpc"
 	"github.com/ethereum/go-ethereum/trie"
@@ -368,7 +369,9 @@ func (beacon *Beacon) Finalize(chain consensus.ChainHeaderReader, header *types.
 	if chain.Config().IsPrague(header.Number, header.Time) {
 		fmt.Println("at block", header.Number, "performing transition?", state.Database().InTransition())
 		parent := chain.GetHeaderByHash(header.ParentHash)
-		overlay.OverlayVerkleTransition(state, parent.Root, chain.Config().OverlayStride)
+		if err := overlay.OverlayVerkleTransition(state, parent.Root, chain.Config().OverlayStride); err != nil {
+			log.Error("error performing the transition", "err", err)
+		}
 	}
 }
 
@@ -394,63 +397,68 @@ func (beacon *Beacon) FinalizeAndAssemble(chain consensus.ChainHeaderReader, hea
 
 	// Assign the final state root to header.
 	header.Root = state.IntermediateRoot(true)
+	state.Database().SaveTransitionState(header.Root)
 
 	var (
 		p    *verkle.VerkleProof
 		k    verkle.StateDiff
 		keys = state.Witness().Keys()
 	)
-	if chain.Config().IsPrague(header.Number, header.Time) && chain.Config().ProofInBlocks {
+	if chain.Config().IsPrague(header.Number, header.Time) {
 		// Open the pre-tree to prove the pre-state against
 		parent := chain.GetHeaderByNumber(header.Number.Uint64() - 1)
 		if parent == nil {
 			return nil, fmt.Errorf("nil parent header for block %d", header.Number)
 		}
 
-		preTrie, err := state.Database().OpenTrie(parent.Root)
-		if err != nil {
-			return nil, fmt.Errorf("error opening pre-state tree root: %w", err)
-		}
+		state.Database().LoadTransitionState(parent.Root)
+
+		if chain.Config().ProofInBlocks {
+			preTrie, err := state.Database().OpenTrie(parent.Root)
+			if err != nil {
+				return nil, fmt.Errorf("error opening pre-state tree root: %w", err)
+			}
 
-		var okpre, okpost bool
-		var vtrpre, vtrpost *trie.VerkleTrie
-		switch pre := preTrie.(type) {
-		case *trie.VerkleTrie:
-			vtrpre, okpre = preTrie.(*trie.VerkleTrie)
-			switch tr := state.GetTrie().(type) {
+			var okpre, okpost bool
+			var vtrpre, vtrpost *trie.VerkleTrie
+			switch pre := preTrie.(type) {
 			case *trie.VerkleTrie:
-				vtrpost = tr
-				okpost = true
-			// This is to handle a situation right at the start of the conversion:
-			// the post trie is a transition tree when the pre tree is an empty
-			// verkle tree.
+				vtrpre, okpre = preTrie.(*trie.VerkleTrie)
+				switch tr := state.GetTrie().(type) {
+				case *trie.VerkleTrie:
+					vtrpost = tr
+					okpost = true
+				// This is to handle a situation right at the start of the conversion:
+				// the post trie is a transition tree when the pre tree is an empty
+				// verkle tree.
+				case *trie.TransitionTrie:
+					vtrpost = tr.Overlay()
+					okpost = true
+				default:
+					okpost = false
+				}
 			case *trie.TransitionTrie:
-				vtrpost = tr.Overlay()
+				vtrpre = pre.Overlay()
+				okpre = true
+				post, _ := state.GetTrie().(*trie.TransitionTrie)
+				vtrpost = post.Overlay()
 				okpost = true
 			default:
-				okpost = false
+				// This should only happen for the first block of the
+				// conversion, when the previous tree is a merkle tree.
+				//  Logically, the "previous" verkle tree is an empty tree.
+				okpre = true
+				vtrpre = trie.NewVerkleTrie(verkle.New(), state.Database().TrieDB(), utils.NewPointCache(), false)
+				post := state.GetTrie().(*trie.TransitionTrie)
+				vtrpost = post.Overlay()
+				okpost = true
 			}
-		case *trie.TransitionTrie:
-			vtrpre = pre.Overlay()
-			okpre = true
-			post, _ := state.GetTrie().(*trie.TransitionTrie)
-			vtrpost = post.Overlay()
-			okpost = true
-		default:
-			// This should only happen for the first block of the
-			// conversion, when the previous tree is a merkle tree.
-			//  Logically, the "previous" verkle tree is an empty tree.
-			okpre = true
-			vtrpre = trie.NewVerkleTrie(verkle.New(), state.Database().TrieDB(), utils.NewPointCache(), false)
-			post := state.GetTrie().(*trie.TransitionTrie)
-			vtrpost = post.Overlay()
-			okpost = true
-		}
-		if okpre && okpost {
-			if len(keys) > 0 {
-				p, k, err = trie.ProveAndSerialize(vtrpre, vtrpost, keys, vtrpre.FlatdbNodeResolver)
-				if err != nil {
-					return nil, fmt.Errorf("error generating verkle proof for block %d: %w", header.Number, err)
+			if okpre && okpost {
+				if len(keys) > 0 {
+					p, k, err = trie.ProveAndSerialize(vtrpre, vtrpost, keys, vtrpre.FlatdbNodeResolver)
+					if err != nil {
+						return nil, fmt.Errorf("error generating verkle proof for block %d: %w", header.Number, err)
+					}
 				}
 			}
 		}

core/blockchain.go

@@ -103,7 +103,7 @@ const (
 	txLookupCacheLimit  = 1024
 	maxFutureBlocks     = 256
 	maxTimeFutureBlocks = 30
-	TriesInMemory       = 128
+	TriesInMemory       = 8192
 
 	// BlockChainVersion ensures that an incompatible database forces a resync from scratch.
 	//
@@ -2010,7 +2010,7 @@ func (bc *BlockChain) insertSideChain(block *types.Block, it *insertIterator) (i
 		parent = bc.GetHeader(parent.ParentHash, parent.Number.Uint64()-1)
 	}
 	if parent == nil {
-		return it.index, errors.New("missing parent")
+		return it.index, fmt.Errorf("missing parent: hash=%x, number=%d", current.Hash(), current.Number)
 	}
 	// Import all the pruned blocks to make the state available
 	var (
@@ -2071,7 +2071,7 @@ func (bc *BlockChain) recoverAncestors(block *types.Block) (common.Hash, error)
 		}
 	}
 	if parent == nil {
-		return common.Hash{}, errors.New("missing parent")
+		return common.Hash{}, fmt.Errorf("missing parent during ancestor recovery: hash=%x, number=%d", block.ParentHash(), block.Number())
 	}
 	// Import all the pruned blocks to make the state available
 	for i := len(hashes) - 1; i >= 0; i-- {
@@ -2309,6 +2309,7 @@ func (bc *BlockChain) SetCanonical(head *types.Block) (common.Hash, error) {
 	defer bc.chainmu.Unlock()
 
 	// Re-execute the reorged chain in case the head state is missing.
+	log.Trace("looking for state", "root", head.Root(), "has state", bc.HasState(head.Root()))
 	if !bc.HasState(head.Root()) {
 		if latestValidHash, err := bc.recoverAncestors(head); err != nil {
 			return latestValidHash, err

core/overlay/conversion.go

@@ -47,7 +47,7 @@ var zeroTreeIndex uint256.Int
 // collect the key-values in a way that is efficient.
 type keyValueMigrator struct {
 	// leafData contains the values for the future leaf for a particular VKT branch.
-	leafData []migratedKeyValue
+	leafData map[branchKey]*migratedKeyValue
 
 	// When prepare() is called, it will start a background routine that will process the leafData
 	// saving the result in newLeaves to be used by migrateCollectedKeyValues(). The background
@@ -66,7 +66,7 @@ func newKeyValueMigrator() *keyValueMigrator {
 	_ = verkle.GetConfig()
 	return &keyValueMigrator{
 		processingReady: make(chan struct{}),
-		leafData:        make([]migratedKeyValue, 0, 10_000),
+		leafData:        make(map[branchKey]*migratedKeyValue, 10_000),
 	}
 }
 
@@ -138,19 +138,17 @@ func (kvm *keyValueMigrator) addAccountCode(addr []byte, codeSize uint64, chunks
 }
 
 func (kvm *keyValueMigrator) getOrInitLeafNodeData(bk branchKey) *verkle.BatchNewLeafNodeData {
-	// Remember that keyValueMigration receives actions ordered by (address, subtreeIndex).
-	// This means that we can assume that the last element of leafData is the one that we
-	// are looking for, or that we need to create a new one.
-	if len(kvm.leafData) == 0 || kvm.leafData[len(kvm.leafData)-1].branchKey != bk {
-		kvm.leafData = append(kvm.leafData, migratedKeyValue{
-			branchKey: bk,
-			leafNodeData: verkle.BatchNewLeafNodeData{
-				Stem:   nil, // It will be calculated in the prepare() phase, since it's CPU heavy.
-				Values: make(map[byte][]byte),
-			},
-		})
+	if ld, ok := kvm.leafData[bk]; ok {
+		return &ld.leafNodeData
 	}
-	return &kvm.leafData[len(kvm.leafData)-1].leafNodeData
+	kvm.leafData[bk] = &migratedKeyValue{
+		branchKey: bk,
+		leafNodeData: verkle.BatchNewLeafNodeData{
+			Stem:   nil, // It will be calculated in the prepare() phase, since it's CPU heavy.
+			Values: make(map[byte][]byte, 256),
+		},
+	}
+	return &kvm.leafData[bk].leafNodeData
 }
 
 func (kvm *keyValueMigrator) prepare() {
@@ -159,6 +157,10 @@ func (kvm *keyValueMigrator) prepare() {
 	go func() {
 		// Step 1: We split kvm.leafData in numBatches batches, and we process each batch in a separate goroutine.
 		//         This fills each leafNodeData.Stem with the correct value.
+		leafData := make([]migratedKeyValue, 0, len(kvm.leafData))
+		for _, v := range kvm.leafData {
+			leafData = append(leafData, *v)
+		}
 		var wg sync.WaitGroup
 		batchNum := runtime.NumCPU()
 		batchSize := (len(kvm.leafData) + batchNum - 1) / batchNum
@@ -170,7 +172,7 @@ func (kvm *keyValueMigrator) prepare() {
 			}
 			wg.Add(1)
 
-			batch := kvm.leafData[start:end]
+			batch := leafData[start:end]
 			go func() {
 				defer wg.Done()
 				var currAddr common.Address
@@ -190,8 +192,8 @@ func (kvm *keyValueMigrator) prepare() {
 
 		// Step 2: Now that we have all stems (i.e: tree keys) calculated, we can create the new leaves.
 		nodeValues := make([]verkle.BatchNewLeafNodeData, len(kvm.leafData))
-		for i := range kvm.leafData {
-			nodeValues[i] = kvm.leafData[i].leafNodeData
+		for i := range leafData {
+			nodeValues[i] = leafData[i].leafNodeData
 		}
 
 		// Create all leaves in batch mode so we can optimize cryptography operations.
@@ -306,7 +308,12 @@ func OverlayVerkleTransition(statedb *state.StateDB, root common.Hash, maxMovedC
 				if err != nil {
 					return err
 				}
-				stIt.Next()
+				processed := stIt.Next()
+				if processed {
+					log.Debug("account has storage and a next item")
+				} else {
+					log.Debug("account has storage and NO next item")
+				}
 
 				// fdb.StorageProcessed will be initialized to `true` if the
 				// entire storage for an account was not entirely processed
@@ -315,6 +322,7 @@ func OverlayVerkleTransition(statedb *state.StateDB, root common.Hash, maxMovedC
 				// If the entire storage was processed, then the iterator was
 				// created in vain, but it's ok as this will not happen often.
 				for ; !migrdb.GetStorageProcessed() && count < maxMovedCount; count++ {
+					log.Trace("Processing storage", "count", count, "slot", stIt.Slot(), "storage processed", migrdb.GetStorageProcessed(), "current account", migrdb.GetCurrentAccountAddress(), "current account hash", migrdb.GetCurrentAccountHash())
 					var (
 						value     []byte   // slot value after RLP decoding
 						safeValue [32]byte // 32-byte aligned value
@@ -337,6 +345,7 @@ func OverlayVerkleTransition(statedb *state.StateDB, root common.Hash, maxMovedC
 							return fmt.Errorf("slotnr len is zero is not 32: %d", len(slotnr))
 						}
 					}
+					log.Trace("found slot number", "number", slotnr)
 					if crypto.Keccak256Hash(slotnr[:]) != stIt.Hash() {
 						return fmt.Errorf("preimage file does not match storage hash: %s!=%s", crypto.Keccak256Hash(slotnr), stIt.Hash())
 					}
@@ -378,6 +387,7 @@ func OverlayVerkleTransition(statedb *state.StateDB, root common.Hash, maxMovedC
 				// Move to the next account, if available - or end
 				// the transition otherwise.
 				if accIt.Next() {
+					log.Trace("Found another account to convert", "hash", accIt.Hash())
 					var addr common.Address
 					if hasPreimagesBin {
 						if _, err := io.ReadFull(fpreimages, addr[:]); err != nil {
@@ -393,6 +403,7 @@ func OverlayVerkleTransition(statedb *state.StateDB, root common.Hash, maxMovedC
 					if crypto.Keccak256Hash(addr[:]) != accIt.Hash() {
 						return fmt.Errorf("preimage file does not match account hash: %s != %s", crypto.Keccak256Hash(addr[:]), accIt.Hash())
 					}
+					log.Trace("Converting account address", "hash", accIt.Hash(), "addr", addr)
 					preimageSeek += int64(len(addr))
 					migrdb.SetCurrentAccountAddress(addr)
 				} else {

core/state/database.go

@@ -340,26 +340,30 @@ func (db *cachingDB) OpenTrie(root common.Hash) (Trie, error) {
 		mpt Trie
 		err error
 	)
+	fmt.Printf("opening trie with root %x, %v %v\n", root, db.InTransition(), db.Transitioned())
 
 	// TODO separate both cases when I can be certain that it won't
 	// find a Verkle trie where is expects a Transitoion trie.
 	if db.InTransition() || db.Transitioned() {
 		// NOTE this is a kaustinen-only change, it will break replay
 		vkt, err := db.openVKTrie(root)
 		if err != nil {
+			log.Error("failed to open the vkt", "err", err)
 			return nil, err
 		}
 
 		// If the verkle conversion has ended, return a single
 		// verkle trie.
 		if db.CurrentTransitionState.ended {
+			log.Debug("transition ended, returning a simple verkle tree")
 			return vkt, nil
 		}
 
 		// Otherwise, return a transition trie, with a base MPT
 		// trie and an overlay, verkle trie.
 		mpt, err = db.openMPTTrie(db.baseRoot)
 		if err != nil {
+			log.Error("failed to open the mpt", "err", err, "root", db.baseRoot)
 			return nil, err
 		}
 
@@ -547,6 +551,8 @@ func (db *cachingDB) SaveTransitionState(root common.Hash) {
 		// Copy so that the address pointer isn't updated after
 		// it has been saved.
 		db.TransitionStatePerRoot[root] = db.CurrentTransitionState.Copy()
+
+		log.Debug("saving transition state", "storage processed", db.CurrentTransitionState.StorageProcessed, "addr", db.CurrentTransitionState.CurrentAccountAddress, "slot hash", db.CurrentTransitionState.CurrentSlotHash, "root", root, "ended", db.CurrentTransitionState.ended, "started", db.CurrentTransitionState.started)
 	}
 }
 
@@ -555,6 +561,9 @@ func (db *cachingDB) LoadTransitionState(root common.Hash) {
 		db.TransitionStatePerRoot = make(map[common.Hash]*TransitionState)
 	}
 
+	// Initialize the first transition state, with the "ended"
+	// field set to true if the database was created
+	// as a verkle database.
 	ts, ok := db.TransitionStatePerRoot[root]
 	if !ok || ts == nil {
 		// Start with a fresh state
@@ -565,5 +574,5 @@ func (db *cachingDB) LoadTransitionState(root common.Hash) {
 	// doesn't get overwritten.
 	db.CurrentTransitionState = ts.Copy()
 
-	fmt.Println("loaded transition state", db.CurrentTransitionState.StorageProcessed)
+	log.Debug("loaded transition state", "storage processed", db.CurrentTransitionState.StorageProcessed, "addr", db.CurrentTransitionState.CurrentAccountAddress, "slot hash", db.CurrentTransitionState.CurrentSlotHash, "root", root, "ended", db.CurrentTransitionState.ended, "started", db.CurrentTransitionState.started)
 }

core/state/statedb.go

@@ -1310,7 +1310,7 @@ func (s *StateDB) Commit(block uint64, deleteEmptyObjects bool) (common.Hash, er
 			// - head layer is paired with HEAD state
 			// - head-1 layer is paired with HEAD-1 state
 			// - head-127 layer(bottom-most diff layer) is paired with HEAD-127 state
-			if err := s.snaps.Cap(root, 128); err != nil {
+			if err := s.snaps.Cap(root, 8192); err != nil {
 				log.Warn("Failed to cap snapshot tree", "root", root, "layers", 128, "err", err)
 			}
 		}

miner/worker.go

@@ -852,7 +852,7 @@ func (w *worker) prepareWork(genParams *generateParams) (*environment, error) {
 	if genParams.parentHash != (common.Hash{}) {
 		block := w.chain.GetBlockByHash(genParams.parentHash)
 		if block == nil {
-			return nil, fmt.Errorf("missing parent")
+			return nil, fmt.Errorf("missing parent: %x", genParams.parentHash)
 		}
 		parent = block.Header()
 	}