Add escape hatch test in e2e test

execution/gethexec/switch_sequencer.go

@@ -116,18 +116,19 @@ func (s *SwitchSequencer) Start(ctx context.Context) error {
 
 	if s.lightClient != nil {
 		s.CallIteratively(func(ctx context.Context) time.Duration {
-			espresso, err := s.lightClient.IsHotShotLive(s.swtichDelayThreshold)
+			isLive, err := s.lightClient.IsHotShotLive(s.swtichDelayThreshold)
 			if err != nil {
-				return 0
+				return s.switchPollInterval
 			}
 
-			if s.IsRunningEspressoMode() && !espresso {
+			if s.IsRunningEspressoMode() && !isLive {
 				err = s.SwitchToCentralized(ctx)
-			} else if !s.IsRunningEspressoMode() && espresso {
+			} else if !s.IsRunningEspressoMode() && isLive {
 				err = s.SwitchToEspresso(ctx)
 			}
 
 			if err != nil {
+				log.Error("Error swithcing mode", "err", err)
 				return 0
 			}
 			return s.switchPollInterval

go.mod

@@ -7,7 +7,7 @@ replace github.com/VictoriaMetrics/fastcache => ./fastcache
 replace github.com/ethereum/go-ethereum => ./go-ethereum
 
 require (
-	github.com/EspressoSystems/espresso-sequencer-go v0.0.20
+	github.com/EspressoSystems/espresso-sequencer-go v0.0.21
 	github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible
 	github.com/Shopify/toxiproxy v2.1.4+incompatible
 	github.com/alicebob/miniredis/v2 v2.32.1

go.sum

@@ -9,6 +9,18 @@ github.com/DataDog/zstd v1.4.5 h1:EndNeuB0l9syBZhut0wns3gV1hL8zX8LIu6ZiVHWLIQ=
 github.com/DataDog/zstd v1.4.5/go.mod h1:1jcaCB/ufaK+sKp1NBhlGmpz41jOoPQ35bpF36t7BBo=
 github.com/EspressoSystems/espresso-sequencer-go v0.0.20 h1:lqxMZm1IKibvTVx9E30DoCbakvJPWMbHVFG43KyIygs=
 github.com/EspressoSystems/espresso-sequencer-go v0.0.20/go.mod h1:BbU8N23RGl45QXSf/bYc8OQ8TG/vlMaPC1GU1acqKmc=
+github.com/EspressoSystems/espresso-sequencer-go v0.0.21-0.20240627061508-b391361a8efd h1:dc/f2PSSwhgtSkUg9yFxoFSomhnBonTiB4BrxmHZ4mI=
+github.com/EspressoSystems/espresso-sequencer-go v0.0.21-0.20240627061508-b391361a8efd/go.mod h1:BbU8N23RGl45QXSf/bYc8OQ8TG/vlMaPC1GU1acqKmc=
+github.com/EspressoSystems/espresso-sequencer-go v0.0.21-0.20240627082021-b15afa4d1e3b h1:nC4ozgrjAsiLIo4LBD+Qsbman6a3Jc7s7WN69mA08Og=
+github.com/EspressoSystems/espresso-sequencer-go v0.0.21-0.20240627082021-b15afa4d1e3b/go.mod h1:BbU8N23RGl45QXSf/bYc8OQ8TG/vlMaPC1GU1acqKmc=
+github.com/EspressoSystems/espresso-sequencer-go v0.0.21-0.20240628053039-6263c8a675aa h1:KhEx/c0MOBqpQZ8NKqQpn39ynk0/Oby5DlB+J8OLhUg=
+github.com/EspressoSystems/espresso-sequencer-go v0.0.21-0.20240628053039-6263c8a675aa/go.mod h1:BbU8N23RGl45QXSf/bYc8OQ8TG/vlMaPC1GU1acqKmc=
+github.com/EspressoSystems/espresso-sequencer-go v0.0.21-0.20240628063139-ed58fed9e319 h1:Vr4E9alDak4XImOX2ai+hMJ/SgfjaIcfXYpmXnIMk6M=
+github.com/EspressoSystems/espresso-sequencer-go v0.0.21-0.20240628063139-ed58fed9e319/go.mod h1:BbU8N23RGl45QXSf/bYc8OQ8TG/vlMaPC1GU1acqKmc=
+github.com/EspressoSystems/espresso-sequencer-go v0.0.21-0.20240628072620-351db6a9e261 h1:Jp4m7pWglCuS6vVimUYQpLzgD9uglmLcET87JFmVbGQ=
+github.com/EspressoSystems/espresso-sequencer-go v0.0.21-0.20240628072620-351db6a9e261/go.mod h1:BbU8N23RGl45QXSf/bYc8OQ8TG/vlMaPC1GU1acqKmc=
+github.com/EspressoSystems/espresso-sequencer-go v0.0.21 h1:pHLdf8qfIGMNLX3QjoPoYzJ+LUgKf30ipd+Pxcypsaw=
+github.com/EspressoSystems/espresso-sequencer-go v0.0.21/go.mod h1:BbU8N23RGl45QXSf/bYc8OQ8TG/vlMaPC1GU1acqKmc=
 github.com/Joker/hpp v1.0.0/go.mod h1:8x5n+M1Hp5hC0g8okX3sR3vFQwynaX/UgSOM9MeBKzY=
 github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible h1:1G1pk05UrOh0NlF1oeaaix1x8XzrfjIDK47TY0Zehcw=
 github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible/go.mod h1:r7JcOSlj0wfOMncg0iLm8Leh48TZaKVeNIfJntJ2wa0=

system_tests/espresso-e2e/docker-compose.yaml

@@ -1,7 +1,7 @@
 version: '3.9'
 services:
   espresso-dev-node:
-    image: ghcr.io/espressosystems/espresso-sequencer/espresso-dev-node:jh-dev-nodemusl
+    image: ghcr.io/espressosystems/espresso-sequencer/espresso-dev-node:main
     ports:
       - "$ESPRESSO_SEQUENCER_API_PORT:$ESPRESSO_SEQUENCER_API_PORT"
       - "$ESPRESSO_BUILDER_PORT:$ESPRESSO_BUILDER_PORT"

system_tests/espresso_e2e_test.go

@@ -11,6 +11,7 @@ import (
 	"testing"
 	"time"
 
+	lightclientmock "github.com/EspressoSystems/espresso-sequencer-go/light-client-mock"
 	espressoTypes "github.com/EspressoSystems/espresso-sequencer-go/types"
 	"github.com/ethereum/go-ethereum/accounts/abi"
 	"github.com/ethereum/go-ethereum/accounts/abi/bind"
@@ -33,13 +34,11 @@ import (
 
 var workingDir = "./espresso-e2e"
 
-// light client
-// var lightClientAddress = "0x60571c8f4b52954a24a5e7306d435e951528d963"
-
 // light client proxy
 var lightClientAddress = "0xb075b82c7a23e0994df4793422a1f03dbcf9136f"
 
 var hotShotUrl = "http://127.0.0.1:41000"
+var delayThreshold = 10
 
 var (
 	jitValidationPort = 54320
@@ -83,6 +82,9 @@ func createL2Node(ctx context.Context, t *testing.T, hotshot_url string, builder
 	nodeConfig.DelayedSequencer.Enable = true
 	nodeConfig.Sequencer = true
 	nodeConfig.Espresso = true
+	builder.execConfig.Sequencer.LightClientAddress = lightClientAddress
+	builder.execConfig.Sequencer.SwitchPollInterval = 10 * time.Second
+	builder.execConfig.Sequencer.SwitchDelayThreshold = uint64(delayThreshold)
 	builder.execConfig.Sequencer.Enable = true
 	builder.execConfig.Sequencer.Espresso = true
 	builder.execConfig.Sequencer.EspressoNamespace = builder.chainConfig.ChainID.Uint64()
@@ -355,9 +357,9 @@ func runNodes(ctx context.Context, t *testing.T) (*NodeBuilder, *TestClient, *Bl
 
 	return builder, l2Node, l2Info, func() {
 		cleanL2Node()
-		cleanEspresso()
 		cleanup()
 		cleanValNode()
+		cleanEspresso()
 	}
 }
 
@@ -397,27 +399,8 @@ func TestEspressoE2E(t *testing.T) {
 	})
 	Require(t, err)
 
-	// Make sure it is a totally new account
-	newAccount := "User10"
-	l2Info.GenerateAccount(newAccount)
-	addr := l2Info.GetAddress(newAccount)
-	balance := l2Node.GetBalance(t, addr)
-	if balance.Cmp(big.NewInt(0)) > 0 {
-		Fatal(t, "empty account")
-	}
-
 	// Check if the tx is executed correctly
-	transferAmount := big.NewInt(1e16)
-	tx := l2Info.PrepareTx("Faucet", newAccount, 3e7, transferAmount, nil)
-	err = l2Node.Client.SendTransaction(ctx, tx)
-	log.Info("Sent faucet tx", "hash", tx.Hash().Hex())
-	Require(t, err)
-
-	err = waitForWith(t, ctx, time.Second*300, time.Second*1, func() bool {
-		balance := l2Node.GetBalance(t, addr)
-		log.Info("waiting for balance", "account", newAccount, "addr", addr, "balance", balance)
-		return balance.Cmp(transferAmount) >= 0
-	})
+	err = checkTransferTxOnL2(t, ctx, l2Node, "User10", l2Info)
 	Require(t, err)
 
 	// Remember the number of messages
@@ -439,14 +422,9 @@ func TestEspressoE2E(t *testing.T) {
 	})
 	Require(t, err)
 
-	// Make sure it is a totally new account
 	newAccount2 := "User11"
 	l2Info.GenerateAccount(newAccount2)
 	addr2 := l2Info.GetAddress(newAccount2)
-	balance2 := l2Node.GetBalance(t, addr2)
-	if balance2.Cmp(big.NewInt(0)) > 0 {
-		Fatal(t, "empty account")
-	}
 
 	// Transfer via the delayed inbox
 	delayedTx := l2Info.PrepareTx("Owner", newAccount2, 3e7, transferAmount, nil)
@@ -519,9 +497,9 @@ func TestEspressoE2E(t *testing.T) {
 		}
 
 		i += 1
-		conflict1, err := validatorUtils.FindStakerConflict(&bind.CallOpts{}, builder.L2.ConsensusNode.DeployInfo.Rollup, goodOpts.From, badOpts1.From, big.NewInt(1024))
+		conflict1, err := validatorUtils.FindStakerConflict(nil, builder.L2.ConsensusNode.DeployInfo.Rollup, goodOpts.From, badOpts1.From, big.NewInt(1024))
 		Require(t, err)
-		conflict2, err := validatorUtils.FindStakerConflict(&bind.CallOpts{}, builder.L2.ConsensusNode.DeployInfo.Rollup, goodOpts.From, badOpts2.From, big.NewInt(1024))
+		conflict2, err := validatorUtils.FindStakerConflict(nil, builder.L2.ConsensusNode.DeployInfo.Rollup, goodOpts.From, badOpts2.From, big.NewInt(1024))
 		Require(t, err)
 		condition := staker.ConflictType(conflict1.Ty) == staker.CONFLICT_TYPE_FOUND && staker.ConflictType(conflict2.Ty) == staker.CONFLICT_TYPE_FOUND
 		if !condition {
@@ -538,6 +516,61 @@ func TestEspressoE2E(t *testing.T) {
 
 	checkStaker := os.Getenv("E2E_CHECK_STAKER")
 	if checkStaker == "" {
+		log.Info("Checking the escape hatch")
+		// Start to check the escape hatch
+		address := common.HexToAddress(lightClientAddress)
+
+		txOpts := builder.L1Info.GetDefaultTransactOpts("Faucet", ctx)
+		// Freeze the l1 height
+		err := lightclientmock.FreezeL1Height(t, builder.L1.Client, address, &txOpts)
+		Require(t, err)
+
+		err = waitForWith(t, ctx, 1*time.Minute, 1*time.Second, func() bool {
+			isLive, err := lightclientmock.IsHotShotLive(t, builder.L1.Client, address, uint64(delayThreshold))
+			if err != nil {
+				return false
+			}
+			return !isLive
+		})
+		Require(t, err)
+
+		// Wait for the switch to be totally finished
+		currMsg, err := node.ConsensusNode.TxStreamer.GetMessageCount()
+		Require(t, err)
+		var validatedMsg arbutil.MessageIndex
+		err = waitForWith(t, ctx, 6*time.Minute, 60*time.Second, func() bool {
+
+			validatedCnt := node.ConsensusNode.BlockValidator.Validated(t)
+			if validatedCnt >= currMsg {
+				validatedMsg = validatedCnt
+				return true
+			}
+			return false
+		})
+		Require(t, err)
+
+		err = checkTransferTxOnL2(t, ctx, l2Node, "User12", l2Info)
+		Require(t, err)
+		err = checkTransferTxOnL2(t, ctx, l2Node, "User13", l2Info)
+		Require(t, err)
+
+		err = waitForWith(t, ctx, 3*time.Minute, 20*time.Second, func() bool {
+			validated := node.ConsensusNode.BlockValidator.Validated(t)
+			return validated >= validatedMsg
+		})
+		Require(t, err)
+
+		// Unfreeze the l1 height
+		err = lightclientmock.UnfreezeL1Height(t, builder.L1.Client, address, &txOpts)
+		Require(t, err)
+
+		// Check if the validated count is increasing
+		err = waitForWith(t, ctx, 3*time.Minute, 20*time.Second, func() bool {
+			validated := node.ConsensusNode.BlockValidator.Validated(t)
+			return validated >= validatedMsg+10
+		})
+		Require(t, err)
+
 		return
 	}
 	err = waitForWith(
@@ -576,3 +609,28 @@ func TestEspressoE2E(t *testing.T) {
 		})
 	Require(t, err)
 }
+
+func checkTransferTxOnL2(
+	t *testing.T,
+	ctx context.Context,
+	l2Node *TestClient,
+	account string,
+	l2Info *BlockchainTestInfo,
+) error {
+	l2Info.GenerateAccount(account)
+	transferAmount := big.NewInt(1e16)
+	tx := l2Info.PrepareTx("Faucet", account, 3e7, transferAmount, nil)
+
+	err := l2Node.Client.SendTransaction(ctx, tx)
+	if err != nil {
+		return err
+	}
+
+	addr := l2Info.GetAddress(account)
+
+	return waitForWith(t, ctx, time.Second*300, time.Second*1, func() bool {
+		balance := l2Node.GetBalance(t, addr)
+		log.Info("waiting for balance", "account", account, "addr", addr, "balance", balance)
+		return balance.Cmp(transferAmount) >= 0
+	})
+}

system_tests/espresso_switch_test.go

@@ -53,16 +53,11 @@ func TestEspressoSwitch(t *testing.T) {
 	})
 	Require(t, err)
 
-	// Make sure it is a totally new account
 	newAccount := "User10"
+	// It will panic if this is not a new account
 	l2Info.GenerateAccount(newAccount)
 	addr := l2Info.GetAddress(newAccount)
-	balance := l2Node.GetBalance(t, addr)
-	if balance.Cmp(big.NewInt(0)) > 0 {
-		Fatal(t, "empty account")
-	}
 
-	// Check if the tx is executed correctly
 	transferAmount := big.NewInt(1e16)
 	tx := l2Info.PrepareTx("Faucet", newAccount, 3e7, transferAmount, nil)
 	err = l2Node.Client.SendTransaction(ctx, tx)