test: add tx priority checks to e2e tests

changelog.md

@@ -29,6 +29,7 @@
 ### Tests
 
 * [2047](https://github.com/zeta-chain/node/pull/2047) - fix liquidity cap advanced test
+* [2184](https://github.com/zeta-chain/node/pull/2184) - add tx priority checks to e2e tests
 
 ### Fixes
 

cmd/zetae2e/local/local.go

@@ -331,8 +331,23 @@ func localE2ETest(cmd *cobra.Command, _ []string) {
 		eg.Go(miscTestRoutine(conf, deployerRunner, verbose, e2etests.TestMyTestName))
 	}
 
+	// while tests are executed, monitor blocks in parallel to check if system txs are on top and they have biggest priority
+	txPriorityErrCh := make(chan error, 1)
+	ctx, monitorPriorityCancel := context.WithCancel(context.Background())
+	go MonitorTxPriorityInBlocks(ctx, conf, txPriorityErrCh)
+
 	if err := eg.Wait(); err != nil {
 		deployerRunner.CtxCancel()
+		monitorPriorityCancel()
+		logger.Print("❌ %v", err)
+		logger.Print("❌ e2e tests failed after %s", time.Since(testStartTime).String())
+		os.Exit(1)
+	}
+
+	// if all tests pass, cancel txs priority monitoring and check if tx priority is not correct in some blocks
+	logger.Print("⏳ e2e tests passed, checking tx priority")
+	monitorPriorityCancel()
+	if err := <-txPriorityErrCh; err != nil {
 		logger.Print("❌ %v", err)
 		logger.Print("❌ e2e tests failed after %s", time.Since(testStartTime).String())
 		os.Exit(1)

cmd/zetae2e/local/performance.go

@@ -59,7 +59,7 @@ func ethereumDepositPerformanceRoutine(
 		}
 
 		if err := r.RunE2ETests(tests); err != nil {
-			return fmt.Errorf("misc tests failed: %v", err)
+			return fmt.Errorf("ethereum deposit performance test failed: %v", err)
 		}
 
 		r.Logger.Print("🍾 Ethereum deposit performance test completed in %s", time.Since(startTime).String())
@@ -117,7 +117,7 @@ func ethereumWithdrawPerformanceRoutine(
 		}
 
 		if err := r.RunE2ETests(tests); err != nil {
-			return fmt.Errorf("misc tests failed: %v", err)
+			return fmt.Errorf("ethereum withdraw performance test failed: %v", err)
 		}
 
 		r.Logger.Print("🍾 Ethereum withdraw performance test completed in %s", time.Since(startTime).String())

cmd/zetae2e/local/utils.go

@@ -2,10 +2,13 @@ package local
 
 import (
 	"context"
+	"errors"
+	"strings"
 
 	"path/filepath"
 	"time"
 
+	rpchttp "github.com/cometbft/cometbft/rpc/client/http"
 	sdk "github.com/cosmos/cosmos-sdk/types"
 	ethcommon "github.com/ethereum/go-ethereum/common"
 	"github.com/spf13/cobra"
@@ -100,3 +103,52 @@ func waitKeygenHeight(
 		logger.Info("Last ZetaHeight: %d", response.Height)
 	}
 }
+
+func MonitorTxPriorityInBlocks(ctx context.Context, conf config.Config, errCh chan error) {
+	rpc, err := rpchttp.New(conf.RPCs.ZetaCoreRPC, "/websocket")
+	if err != nil {
+		errCh <- err
+	}
+
+	ticker := time.NewTicker(2 * time.Second)
+	defer ticker.Stop()
+	for {
+		select {
+		case <-ctx.Done():
+			errCh <- nil
+		case <-ticker.C:
+			block, err := rpc.Block(ctx, nil)
+			if err != nil {
+				errCh <- err
+			}
+
+			// iterate txs events and check if some MsgEthereumTx is included above crosschain and observer txs
+			nonSystemTxFound := false
+			for _, tx := range block.Block.Txs {
+				txRes, err := rpc.Tx(context.Background(), tx.Hash(), false)
+				if err != nil {
+					continue
+				}
+
+				for _, ev := range txRes.TxResult.Events {
+					for _, attr := range ev.Attributes {
+						if attr.Key == "msg_type_url" {
+							if strings.Contains(attr.Value, "zetachain.zetacore.crosschain.MsgVote") ||
+								strings.Contains(attr.Value, "zetachain.zetacore.observer.MsgVote") ||
+								strings.Contains(attr.Value, "zetachain.zetacore.observer.MsgAddBlameVote") {
+								if nonSystemTxFound {
+									errCh <- errors.New("wrong tx priority, system tx not on top")
+								}
+							}
+						}
+						if attr.Key == "action" {
+							if strings.Contains(attr.Value, "ethermint.evm.v1.MsgEthereumTx") {
+								nonSystemTxFound = true
+							}
+						}
+					}
+				}
+			}
+		}
+	}
+}