Refactor transition post genesis (#311)

* rewrite per-block conversion pointer management

* remove unused method

* fix: a branch that can verge at genesis or post genesis (#314)

* fix: import cycle in conversion refactor (#315)

* fix shadowfork panic in OpenStorageTrie

cmd/geth/chaincmd.go

@@ -214,6 +214,7 @@ func initGenesis(ctx *cli.Context) error {
 		}
 		triedb := trie.NewDatabaseWithConfig(chaindb, &trie.Config{
 			Preimages: ctx.Bool(utils.CachePreimagesFlag.Name),
+			Verkle:    genesis.IsVerkle(),
 		})
 		_, hash, err := core.SetupGenesisBlockWithOverride(chaindb, triedb, genesis, &overrides)
 		if err != nil {

consensus/beacon/consensus.go

@@ -25,6 +25,7 @@ import (
 	"github.com/ethereum/go-ethereum/consensus"
 	"github.com/ethereum/go-ethereum/consensus/misc/eip1559"
 	"github.com/ethereum/go-ethereum/consensus/misc/eip4844"
+	"github.com/ethereum/go-ethereum/core/overlay"
 	"github.com/ethereum/go-ethereum/core/state"
 	"github.com/ethereum/go-ethereum/core/types"
 	"github.com/ethereum/go-ethereum/params"
@@ -368,6 +369,12 @@ func (beacon *Beacon) Finalize(chain consensus.ChainHeaderReader, header *types.
 	state.Witness().TouchAddressOnWriteAndComputeGas(header.Coinbase[:], uint256.Int{}, utils.NonceLeafKey)
 	state.Witness().TouchAddressOnWriteAndComputeGas(header.Coinbase[:], uint256.Int{}, utils.CodeKeccakLeafKey)
 	state.Witness().TouchAddressOnWriteAndComputeGas(header.Coinbase[:], uint256.Int{}, utils.CodeSizeLeafKey)
+
+	if chain.Config().IsPrague(header.Number, header.Time) {
+		fmt.Println("at block", header.Number, "performing transition?", state.Database().InTransition())
+		parent := chain.GetHeaderByHash(header.ParentHash)
+		overlay.OverlayVerkleTransition(state, parent.Root)
+	}
 }
 
 // FinalizeAndAssemble implements consensus.Engine, setting the final state and
@@ -392,6 +399,7 @@ func (beacon *Beacon) FinalizeAndAssemble(chain consensus.ChainHeaderReader, hea
 
 	// Assign the final state root to header.
 	header.Root = state.IntermediateRoot(true)
+	state.Database().SaveTransitionState(header.Root)
 
 	var (
 		p    *verkle.VerkleProof
@@ -405,6 +413,7 @@ func (beacon *Beacon) FinalizeAndAssemble(chain consensus.ChainHeaderReader, hea
 			return nil, fmt.Errorf("nil parent header for block %d", header.Number)
 		}
 
+		state.Database().LoadTransitionState(parent.Root)
 		preTrie, err := state.Database().OpenTrie(parent.Root)
 		if err != nil {
 			return nil, fmt.Errorf("error opening pre-state tree root: %w", err)
@@ -435,7 +444,14 @@ func (beacon *Beacon) FinalizeAndAssemble(chain consensus.ChainHeaderReader, hea
 			vtrpost = post.Overlay()
 			okpost = true
 		default:
-			panic("invalid tree type")
+			// This should only happen for the first block,
+			// so the previous tree is a merkle tree. Logically,
+			// the "previous" verkle tree is an empty tree.
+			okpre = true
+			vtrpre = trie.NewVerkleTrie(verkle.New(), state.Database().TrieDB(), utils.NewPointCache(), false)
+			post := state.GetTrie().(*trie.TransitionTrie)
+			vtrpost = post.Overlay()
+			okpost = true
 		}
 		if okpre && okpost {
 			if len(keys) > 0 {

core/blockchain.go

@@ -312,6 +312,12 @@ func NewBlockChain(db ethdb.Database, cacheConfig *CacheConfig, genesis *Genesis
 
 	// Declare the end of the verkle transition if need be
 	if bc.chainConfig.Rules(head.Number, false /* XXX */, head.Time).IsPrague {
+		// TODO this only works when resuming a chain that has already gone
+		// through the conversion. All pointers should be saved to the DB
+		// for it to be able to recover if interrupted during the transition
+   // but that's left out to a later PR since there's not really a need
+   // right now.
+		bc.stateCache.InitTransitionStatus(true, true)
 		bc.stateCache.EndVerkleTransition()
 	}
 

core/chain_makers.go

@@ -425,13 +425,15 @@ func GenerateVerkleChain(config *params.ChainConfig, parent *types.Block, engine
 		return nil, nil
 	}
 	var snaps *snapshot.Tree
+	triedb := state.NewDatabaseWithConfig(db, nil)
+	triedb.StartVerkleTransition(common.Hash{}, common.Hash{}, config, config.PragueTime, common.Hash{})
+	triedb.EndVerkleTransition()
+	statedb, err := state.New(parent.Root(), triedb, snaps)
+	if err != nil {
+		panic(fmt.Sprintf("could not find state for block %d: err=%v, parent root=%x", parent.NumberU64(), err, parent.Root()))
+	}
+	statedb.Database().SaveTransitionState(parent.Root())
 	for i := 0; i < n; i++ {
-		triedb := state.NewDatabaseWithConfig(db, nil)
-		triedb.EndVerkleTransition()
-		statedb, err := state.New(parent.Root(), triedb, snaps)
-		if err != nil {
-			panic(fmt.Sprintf("could not find state for block %d: err=%v, parent root=%x", i, err, parent.Root()))
-		}
 		block, receipt := genblock(i, parent, statedb)
 		blocks[i] = block
 		receipts[i] = receipt

core/genesis.go

@@ -126,6 +126,7 @@ func (ga *GenesisAlloc) deriveHash(cfg *params.ChainConfig, timestamp uint64) (c
 	// all the derived states will be discarded to not pollute disk.
 	db := state.NewDatabase(rawdb.NewMemoryDatabase())
 	if cfg.IsPrague(big.NewInt(int64(0)), timestamp) {
+		db.StartVerkleTransition(common.Hash{}, common.Hash{}, cfg, &timestamp, common.Hash{})
 		db.EndVerkleTransition()
 	}
 	statedb, err := state.New(types.EmptyRootHash, db, nil)
@@ -146,15 +147,17 @@ func (ga *GenesisAlloc) deriveHash(cfg *params.ChainConfig, timestamp uint64) (c
 // flush is very similar with deriveHash, but the main difference is
 // all the generated states will be persisted into the given database.
 // Also, the genesis state specification will be flushed as well.
-func (ga *GenesisAlloc) flush(db ethdb.Database, triedb *trie.Database, blockhash common.Hash, cfg *params.ChainConfig) error {
-	statedb, err := state.New(types.EmptyRootHash, state.NewDatabaseWithNodeDB(db, triedb), nil)
-	if err != nil {
-		return err
+func (ga *GenesisAlloc) flush(db ethdb.Database, triedb *trie.Database, blockhash common.Hash, cfg *params.ChainConfig, timestamp *uint64) error {
+	database := state.NewDatabaseWithNodeDB(db, triedb)
+	// End the verkle conversion at genesis if the fork block is 0
+	if timestamp != nil && cfg.IsPrague(big.NewInt(int64(0)), *timestamp) {
+		database.StartVerkleTransition(common.Hash{}, common.Hash{}, cfg, timestamp, common.Hash{})
+		database.EndVerkleTransition()
 	}
 
-	// End the verkle conversion at genesis if the fork block is 0
-	if triedb.IsVerkle() {
-		statedb.Database().EndVerkleTransition()
+	statedb, err := state.New(types.EmptyRootHash, database, nil)
+	if err != nil {
+		return err
 	}
 
 	for addr, account := range *ga {
@@ -221,7 +224,7 @@ func CommitGenesisState(db ethdb.Database, triedb *trie.Database, blockhash comm
 			return errors.New("not found")
 		}
 	}
-	return alloc.flush(db, triedb, blockhash, config)
+	return alloc.flush(db, triedb, blockhash, config, nil)
 }
 
 // GenesisAccount is an account in the state of the genesis block.
@@ -456,6 +459,12 @@ func (g *Genesis) configOrDefault(ghash common.Hash) *params.ChainConfig {
 	}
 }
 
+// IsVerkle indicates whether the state is already stored in a verkle
+// tree at genesis time.
+func (g *Genesis) IsVerkle() bool {
+	return g.Config.IsPrague(new(big.Int).SetUint64(g.Number), g.Timestamp)
+}
+
 // ToBlock returns the genesis block according to genesis specification.
 func (g *Genesis) ToBlock() *types.Block {
 	root, err := g.Alloc.deriveHash(g.Config, g.Timestamp)
@@ -530,7 +539,7 @@ func (g *Genesis) Commit(db ethdb.Database, triedb *trie.Database) (*types.Block
 	// All the checks has passed, flush the states derived from the genesis
 	// specification as well as the specification itself into the provided
 	// database.
-	if err := g.Alloc.flush(db, triedb, block.Hash(), g.Config); err != nil {
+	if err := g.Alloc.flush(db, triedb, block.Hash(), g.Config, &g.Timestamp); err != nil {
 		return nil, err
 	}
 	rawdb.WriteTd(db, block.Hash(), block.NumberU64(), block.Difficulty())