Added tool for processing for all mainnet beacon blocks (#7095)

cl/clparams/config.go

@@ -43,8 +43,8 @@ const (
 	MaxDialTimeout               = 2 * time.Second
 	VersionLength  int           = 4
 	MaxChunkSize   uint64        = 1 << 20 // 1 MiB
-	ReqTimeout     time.Duration = 1 * time.Second
-	RespTimeout    time.Duration = 1 * time.Second
+	ReqTimeout     time.Duration = 10 * time.Second
+	RespTimeout    time.Duration = 15 * time.Second
 )
 
 var (

cl/clparams/initial_state/initial_state.go

@@ -0,0 +1,42 @@
+package initial_state
+
+import (
+	_ "embed"
+	"fmt"
+
+	"github.com/ledgerwatch/erigon/cl/clparams"
+	"github.com/ledgerwatch/erigon/cmd/erigon-cl/core/state"
+)
+
+//go:embed mainnet.state.ssz
+var mainnetStateSSZ []byte
+
+//go:embed sepolia.state.ssz
+var sepoliaStateSSZ []byte
+
+//go:embed goerli.state.ssz
+var goerliStateSSZ []byte
+
+// Return genesis state
+func GetGenesisState(network clparams.NetworkType) (*state.BeaconState, error) {
+	_, _, config := clparams.GetConfigsByNetwork(network)
+	returnState := state.New(config)
+
+	switch network {
+	case clparams.MainnetNetwork:
+		if err := returnState.DecodeSSZWithVersion(mainnetStateSSZ, int(clparams.Phase0Version)); err != nil {
+			return nil, err
+		}
+	case clparams.GoerliNetwork:
+		if err := returnState.DecodeSSZWithVersion(goerliStateSSZ, int(clparams.Phase0Version)); err != nil {
+			return nil, err
+		}
+	case clparams.SepoliaNetwork:
+		if err := returnState.DecodeSSZWithVersion(sepoliaStateSSZ, int(clparams.Phase0Version)); err != nil {
+			return nil, err
+		}
+	default:
+		return nil, fmt.Errorf("unsupported network for genesis fetching")
+	}
+	return returnState, nil
+}

cl/clparams/initial_state/initial_state_test.go

@@ -0,0 +1,34 @@
+package initial_state_test
+
+import (
+	"testing"
+
+	libcommon "github.com/ledgerwatch/erigon-lib/common"
+	"github.com/ledgerwatch/erigon/cl/clparams"
+	"github.com/ledgerwatch/erigon/cl/clparams/initial_state"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestMainnet(t *testing.T) {
+	state, err := initial_state.GetGenesisState(clparams.MainnetNetwork)
+	assert.NoError(t, err)
+	root, err := state.HashSSZ()
+	assert.NoError(t, err)
+	assert.Equal(t, libcommon.Hash(root), libcommon.HexToHash("7e76880eb67bbdc86250aa578958e9d0675e64e714337855204fb5abaaf82c2b"))
+}
+
+func TestGoerli(t *testing.T) {
+	state, err := initial_state.GetGenesisState(clparams.GoerliNetwork)
+	assert.NoError(t, err)
+	root, err := state.HashSSZ()
+	assert.NoError(t, err)
+	assert.Equal(t, libcommon.Hash(root), libcommon.HexToHash("895390e92edc03df7096e9f51e51896e8dbe6e7e838180dadbfd869fdd77a659"))
+}
+
+func TestSepolia(t *testing.T) {
+	state, err := initial_state.GetGenesisState(clparams.SepoliaNetwork)
+	assert.NoError(t, err)
+	root, err := state.HashSSZ()
+	assert.NoError(t, err)
+	assert.Equal(t, libcommon.Hash(root), libcommon.HexToHash("fb9afe32150fa39f4b346be2519a67e2a4f5efcd50a1dc192c3f6b3d013d2798"))
+}

cmd/erigon-cl/main.go

@@ -10,6 +10,7 @@ import (
 	"github.com/ledgerwatch/erigon-lib/kv"
 	"github.com/ledgerwatch/erigon-lib/kv/mdbx"
 	"github.com/ledgerwatch/erigon/cl/clparams"
+	"github.com/ledgerwatch/erigon/cl/clparams/initial_state"
 	"github.com/ledgerwatch/erigon/cl/cltypes"
 	"github.com/ledgerwatch/erigon/cl/fork"
 	"github.com/ledgerwatch/erigon/cl/rpc"
@@ -20,6 +21,7 @@ import (
 	"github.com/ledgerwatch/erigon/cmd/erigon-cl/network"
 	"github.com/ledgerwatch/erigon/cmd/erigon-cl/stages"
 	lcCli "github.com/ledgerwatch/erigon/cmd/sentinel/cli"
+
 	"github.com/ledgerwatch/erigon/cmd/sentinel/cli/flags"
 	"github.com/ledgerwatch/erigon/cmd/sentinel/sentinel"
 	"github.com/ledgerwatch/erigon/cmd/sentinel/sentinel/handshake"
@@ -58,12 +60,8 @@ func runConsensusLayerNode(cliCtx *cli.Context) error {
 		log.Error("Could load beacon data configuration", "err", err)
 		return err
 	}
-	// Fetch the checkpoint state.
-	cpState, err := getCheckpointState(ctx, db, cfg.BeaconCfg, cfg.GenesisCfg, cfg.CheckpointUri)
-	if err != nil {
-		log.Error("Could not get checkpoint", "err", err)
-		return err
-	}
+
+	tmpdir := "/tmp"
 	var executionClient *execution_client.ExecutionClient
 	if cfg.ELEnabled {
 		executionClient, err = execution_client.NewExecutionClient(ctx, "127.0.0.1:8989")
@@ -73,8 +71,24 @@ func runConsensusLayerNode(cliCtx *cli.Context) error {
 		}
 	}
 
+	if cfg.TransitionChain {
+		state, err := initial_state.GetGenesisState(cfg.NetworkType)
+		if err != nil {
+			return err
+		}
+		// Execute from genesis to whatever we have.
+		return stages.SpawnStageBeaconState(stages.StageBeaconState(db, cfg.BeaconCfg, state, nil, true, executionClient), nil, ctx)
+	}
+
+	fmt.Println(cfg.CheckpointUri)
+	// Fetch the checkpoint state.
+	cpState, err := getCheckpointState(ctx, db, cfg.BeaconCfg, cfg.GenesisCfg, cfg.CheckpointUri)
+	if err != nil {
+		log.Error("Could not get checkpoint", "err", err)
+		return err
+	}
+
 	log.Info("Starting sync from checkpoint.")
-	tmpdir := "/tmp"
 	// Start the sentinel service
 	log.Root().SetHandler(log.LvlFilterHandler(log.Lvl(cfg.LogLvl), log.StderrHandler))
 	log.Info("[Sentinel] running sentinel with configuration", "cfg", cfg)

cmd/erigon-cl/stages/stages.go

@@ -40,7 +40,7 @@ func ConsensusStages(ctx context.Context, historyReconstruction StageHistoryReco
 			ID:          stages.BeaconState,
 			Description: "Execute Consensus Layer transition",
 			Forward: func(firstCycle bool, badBlockUnwind bool, s *stagedsync.StageState, u stagedsync.Unwinder, tx kv.RwTx, quiet bool) error {
-				return SpawnStageBeaconState(beaconState, s, tx, ctx)
+				return SpawnStageBeaconState(beaconState, tx, ctx)
 			},
 			Unwind: func(firstCycle bool, u *stagedsync.UnwindState, s *stagedsync.StageState, tx kv.RwTx) error {
 				return nil
@@ -77,7 +77,7 @@ func NewConsensusStagedSync(ctx context.Context,
 			ctx,
 			StageHistoryReconstruction(db, backwardDownloader, genesisCfg, beaconCfg, beaconDBCfg, state, tmpdir, executionClient),
 			StageBeaconsBlock(db, forwardDownloader, genesisCfg, beaconCfg, state, executionClient),
-			StageBeaconState(db, genesisCfg, beaconCfg, state, triggerExecution, clearEth1Data, executionClient),
+			StageBeaconState(db, beaconCfg, state, triggerExecution, clearEth1Data, executionClient),
 		),
 		ConsensusUnwindOrder,
 		ConsensusPruneOrder,

cmd/erigon-cl/stages/stages_beacon_state.go

@@ -2,7 +2,6 @@ package stages
 
 import (
 	"context"
-	"fmt"
 
 	libcommon "github.com/ledgerwatch/erigon-lib/common"
 	"github.com/ledgerwatch/erigon-lib/kv"
@@ -14,7 +13,6 @@ import (
 	"github.com/ledgerwatch/erigon/cmd/erigon-cl/core/state"
 	"github.com/ledgerwatch/erigon/cmd/erigon-cl/core/transition"
 	"github.com/ledgerwatch/erigon/cmd/erigon-cl/execution_client"
-	"github.com/ledgerwatch/erigon/eth/stagedsync"
 	"github.com/ledgerwatch/erigon/eth/stagedsync/stages"
 	"github.com/ledgerwatch/log/v3"
 )
@@ -24,19 +22,17 @@ type triggerExecutionFunc func(*cltypes.SignedBeaconBlock) error
 
 type StageBeaconStateCfg struct {
 	db               kv.RwDB
-	genesisCfg       *clparams.GenesisConfig
 	beaconCfg        *clparams.BeaconChainConfig
 	state            *state.BeaconState
 	clearEth1Data    bool // Whether we want to discard eth1 data.
 	triggerExecution triggerExecutionFunc
 	executionClient  *execution_client.ExecutionClient
 }
 
-func StageBeaconState(db kv.RwDB, genesisCfg *clparams.GenesisConfig,
+func StageBeaconState(db kv.RwDB,
 	beaconCfg *clparams.BeaconChainConfig, state *state.BeaconState, triggerExecution triggerExecutionFunc, clearEth1Data bool, executionClient *execution_client.ExecutionClient) StageBeaconStateCfg {
 	return StageBeaconStateCfg{
 		db:               db,
-		genesisCfg:       genesisCfg,
 		beaconCfg:        beaconCfg,
 		state:            state,
 		clearEth1Data:    clearEth1Data,
@@ -46,7 +42,7 @@ func StageBeaconState(db kv.RwDB, genesisCfg *clparams.GenesisConfig,
 }
 
 // SpawnStageBeaconForward spawn the beacon forward stage
-func SpawnStageBeaconState(cfg StageBeaconStateCfg, s *stagedsync.StageState, tx kv.RwTx, ctx context.Context) error {
+func SpawnStageBeaconState(cfg StageBeaconStateCfg, tx kv.RwTx, ctx context.Context) error {
 	useExternalTx := tx != nil
 	var err error
 	if !useExternalTx {
@@ -83,7 +79,7 @@ func SpawnStageBeaconState(cfg StageBeaconStateCfg, s *stagedsync.StageState, tx
 				return err
 			}
 			// validate fully only in current epoch.
-			fullValidate := utils.GetCurrentEpoch(cfg.genesisCfg.GenesisTime, cfg.beaconCfg.SecondsPerSlot, cfg.beaconCfg.SlotsPerEpoch) == cfg.state.Epoch()
+			fullValidate := utils.GetCurrentEpoch(cfg.state.GenesisTime(), cfg.beaconCfg.SecondsPerSlot, cfg.beaconCfg.SlotsPerEpoch) == cfg.state.Epoch()
 			if err := transition.TransitionState(cfg.state, block, fullValidate); err != nil {
 				log.Info("Found epoch, so stopping now...", "count", slot-(fromSlot+1), "slot", slot)
 				return err
@@ -129,7 +125,7 @@ func SpawnStageBeaconState(cfg StageBeaconStateCfg, s *stagedsync.StageState, tx
 		}
 	}
 
-	log.Info(fmt.Sprintf("[%s] Finished transitioning state", s.LogPrefix()), "from", fromSlot, "to", endSlot)
+	log.Info("[BeaconState] Finished transitioning state", "from", fromSlot, "to", endSlot)
 	if !useExternalTx {
 		if err = tx.Commit(); err != nil {
 			return err

cmd/erigon-el-mock/main.go

@@ -1,17 +1,23 @@
 package main
 
 import (
+	"flag"
 	"net"
 
 	"github.com/c2h5oh/datasize"
 	"github.com/ledgerwatch/erigon-lib/gointerfaces/execution"
+	"github.com/ledgerwatch/erigon-lib/kv"
+	"github.com/ledgerwatch/erigon-lib/kv/mdbx"
 	"github.com/ledgerwatch/erigon-lib/kv/memdb"
 	"google.golang.org/grpc"
 
 	"github.com/ledgerwatch/log/v3"
 )
 
 func main() {
+
+	datadir := flag.String("datadir", "", "non in-memory db for EL simulation")
+	flag.Parse()
 	log.Root().SetHandler(log.LvlFilterHandler(log.LvlInfo, log.StderrHandler))
 	lis, err := net.Listen("tcp", "127.0.0.1:8989")
 	if err != nil {
@@ -20,7 +26,17 @@ func main() {
 	maxReceiveSize := 500 * datasize.MB
 
 	s := grpc.NewServer(grpc.MaxRecvMsgSize(int(maxReceiveSize)))
-	execution.RegisterExecutionServer(s, NewEth1Execution(memdb.New("" /* tmpDir */)))
+	var db kv.RwDB
+	if *datadir == "" {
+		db = memdb.New("")
+	} else {
+		db, err = mdbx.Open(*datadir, log.Root(), false)
+		if err != nil {
+			log.Error("Could not open database", "err", err)
+			return
+		}
+	}
+	execution.RegisterExecutionServer(s, NewEth1Execution(db))
 	log.Info("Serving mock Execution layer.")
 	if err := s.Serve(lis); err != nil {
 		log.Error("failed to serve", "err", err)

cmd/sentinel/cli/cliSettings.go

@@ -27,14 +27,15 @@ type ConsensusClientCliCfg struct {
 	Chaindata        string                      `json:"chaindata"`
 	ELEnabled        bool                        `json:"elEnabled"`
 	ErigonPrivateApi string                      `json:"erigonPrivateApi"`
+	TransitionChain  bool                        `json:"transitionChain"`
+	NetworkType      clparams.NetworkType
 }
 
 func SetupConsensusClientCfg(ctx *cli.Context) (*ConsensusClientCliCfg, error) {
 	cfg := &ConsensusClientCliCfg{}
 	chainName := ctx.String(flags.Chain.Name)
 	var err error
-	var network clparams.NetworkType
-	cfg.GenesisCfg, cfg.NetworkCfg, cfg.BeaconCfg, network, err = clparams.GetConfigsByNetworkName(chainName)
+	cfg.GenesisCfg, cfg.NetworkCfg, cfg.BeaconCfg, cfg.NetworkType, err = clparams.GetConfigsByNetworkName(chainName)
 	if err != nil {
 		return nil, err
 	}
@@ -64,7 +65,8 @@ func SetupConsensusClientCfg(ctx *cli.Context) (*ConsensusClientCliCfg, error) {
 	if ctx.String(flags.CheckpointSyncUrlFlag.Name) != "" {
 		cfg.CheckpointUri = ctx.String(flags.CheckpointSyncUrlFlag.Name)
 	} else {
-		cfg.CheckpointUri = clparams.GetCheckpointSyncEndpoint(network)
+		cfg.CheckpointUri = clparams.GetCheckpointSyncEndpoint(cfg.NetworkType)
+		fmt.Println(cfg.CheckpointUri)
 	}
 	cfg.Chaindata = ctx.String(flags.ChaindataFlag.Name)
 	cfg.ELEnabled = ctx.Bool(flags.ELEnabledFlag.Name)
@@ -76,5 +78,6 @@ func SetupConsensusClientCfg(ctx *cli.Context) (*ConsensusClientCliCfg, error) {
 	if ctx.String(flags.SentinelStaticPeersFlag.Name) != "" {
 		cfg.NetworkCfg.StaticPeers = strings.Split(ctx.String(flags.SentinelStaticPeersFlag.Name), ",")
 	}
+	cfg.TransitionChain = ctx.Bool(flags.TransitionChainFlag.Name)
 	return cfg, nil
 }

cmd/sentinel/cli/flags/defaultFlags.go

@@ -39,4 +39,5 @@ var LCDefaultFlags = []cli.Flag{
 	&CheckpointSyncUrlFlag,
 	&SentinelStaticPeersFlag,
 	&ErigonPrivateApiFlag,
+	&TransitionChainFlag,
 }

cmd/sentinel/cli/flags/flags.go

@@ -88,4 +88,8 @@ var (
 		Usage: "connect to comma-separated Consensus static peers",
 		Value: "",
 	}
+	TransitionChainFlag = cli.BoolFlag{
+		Name:  "transition-chain",
+		Usage: "enable chain transition",
+	}
 )

cmd/sentinel/sentinel/request.go

@@ -30,10 +30,8 @@ func (s *Sentinel) SendRequestRaw(data []byte, topic string) ([]byte, bool, erro
 	s.peers.PeerDoRequest(pid)
 	defer s.peers.PeerFinishRequest(pid)
 	data, isError, err := communication.SendRequestRawToPeer(s.ctx, s.host, data, topic, pid)
-	if err != nil || isError {
+	if err != nil {
 		s.peers.Penalize(pid)
-	} else {
-		s.peers.Forgive(pid)
 	}
 	return data, isError, err
 }