test: add import data to e2e tests

Dockerfile-upgrade

@@ -20,8 +20,8 @@ WORKDIR /go/delivery/zeta-node
 RUN mkdir -p  $GOPATH/bin/old
 RUN mkdir -p  $GOPATH/bin/new
 
-ARG OLD_VERSION=v14.0.0
-ENV NEW_VERSION=v15
+ARG OLD_VERSION=v15.0.0
+ENV NEW_VERSION=v16
 
 # Build new release from the current source
 COPY go.mod /go/delivery/zeta-node/

Makefile

@@ -206,10 +206,15 @@ start-e2e-admin-test: zetanode
 	@echo "--> Starting e2e admin test"
 	cd contrib/localnet/ && $(DOCKER) compose -f docker-compose.yml -f docker-compose-admin.yml up -d
 
+start-e2e-import-test: zetanode
+	@echo "--> Starting e2e import-data test"
+	cd contrib/localnet/  && chmod +x scripts/import-data.sh && ./scripts/import-data.sh mainnet && $(DOCKER) compose -f docker-compose.yml -f docker-compose-import-data.yml up -d
+
 start-e2e-performance-test: zetanode
 	@echo "--> Starting e2e performance test"
 	cd contrib/localnet/ && $(DOCKER) compose -f docker-compose.yml -f docker-compose-performance.yml up -d
 
+
 start-stress-test: zetanode
 	@echo "--> Starting stress test"
 	cd contrib/localnet/ && $(DOCKER) compose -f docker-compose.yml -f docker-compose-stresstest.yml up -d

app/setup_handlers.go

@@ -10,7 +10,7 @@ import (
 	observertypes "github.com/zeta-chain/zetacore/x/observer/types"
 )
 
-const releaseVersion = "v15"
+const releaseVersion = "v16"
 
 func SetupHandlers(app *App) {
 	app.UpgradeKeeper.SetUpgradeHandler(releaseVersion, func(ctx sdk.Context, _ types.Plan, vm module.VersionMap) (module.VersionMap, error) {

changelog.md

@@ -83,6 +83,7 @@
 * [1985](https://github.com/zeta-chain/node/pull/1985) - improve fungible module coverage
 * [1992](https://github.com/zeta-chain/node/pull/1992) - remove setupKeeper from crosschain module
 * [2008](https://github.com/zeta-chain/node/pull/2008) - add test for connector bytecode update
+* [2127](https://github.com/zeta-chain/node/pull/2127) - add import data to e2e test
 
 ### Fixes
 

cmd/zetacored/parse_genesis.go

@@ -47,9 +47,7 @@ var Copy = map[string]bool{
 	upgradetypes.ModuleName:   true,
 	evidencetypes.ModuleName:  true,
 	vestingtypes.ModuleName:   true,
-	fungibletypes.ModuleName:  true,
 	emissionstypes.ModuleName: true,
-	authz.ModuleName:          true,
 }
 
 // Skip represents a set of modules for which, the entire state is skipped and nothing gets imported
@@ -61,6 +59,10 @@ var Skip = map[string]bool{
 	banktypes.ModuleName:         true,
 	distributiontypes.ModuleName: true,
 	group.ModuleName:             true,
+	// Skipping authz as it is not used when starting a new chain, new grants should be created based on the validator hotkeys abd operator keys
+	authz.ModuleName: true,
+	// Skipping fungible module as new fungible tokens would be created and system contract would be deployed
+	fungibletypes.ModuleName: true,
 }
 
 // Modify represents a set of modules for which, the state is modified before importing. Each Module should have a corresponding Modify function
@@ -140,7 +142,6 @@ func ImportDataIntoFile(genDoc *types.GenesisDoc, importFile *types.GenesisDoc,
 			}
 		}
 	}
-
 	appStateJSON, err := json.Marshal(appState)
 	if err != nil {
 		return fmt.Errorf("failed to marshal application genesis state: %w", err)
@@ -153,15 +154,17 @@ func ImportDataIntoFile(genDoc *types.GenesisDoc, importFile *types.GenesisDoc,
 // ModifyCrosschainState modifies the crosschain state before importing
 // It truncates the crosschain transactions, inbound transactions and finalized inbounds to MaxItemsForList
 func ModifyCrosschainState(appState map[string]json.RawMessage, importAppState map[string]json.RawMessage, cdc codec.Codec) error {
-	importedCrossChainGenState := crosschaintypes.GetGenesisStateFromAppState(cdc, importAppState)
-	importedCrossChainGenState.CrossChainTxs = importedCrossChainGenState.CrossChainTxs[:math.Min(MaxItemsForList, len(importedCrossChainGenState.CrossChainTxs))]
-	importedCrossChainGenState.InTxHashToCctxList = importedCrossChainGenState.InTxHashToCctxList[:math.Min(MaxItemsForList, len(importedCrossChainGenState.InTxHashToCctxList))]
-	importedCrossChainGenState.FinalizedInbounds = importedCrossChainGenState.FinalizedInbounds[:math.Min(MaxItemsForList, len(importedCrossChainGenState.FinalizedInbounds))]
-	importedCrossChainStateBz, err := json.Marshal(importedCrossChainGenState)
+	importedCrossChainGenState := crosschaintypes.GetGenesisStateFromAppStateLegacy(cdc, importAppState)
+	appStateGenState := crosschaintypes.GetGenesisStateFromAppState(cdc, appState)
+	// The genesis state has been modified between the two versions , so we add only the required fields and leave out the rest
+	appStateGenState.CrossChainTxs = importedCrossChainGenState.CrossChainTxs[:math.Min(MaxItemsForList, len(importedCrossChainGenState.CrossChainTxs))]
+	appStateGenState.InTxHashToCctxList = importedCrossChainGenState.InTxHashToCctxList[:math.Min(MaxItemsForList, len(importedCrossChainGenState.InTxHashToCctxList))]
+	appStateGenState.FinalizedInbounds = importedCrossChainGenState.FinalizedInbounds[:math.Min(MaxItemsForList, len(importedCrossChainGenState.FinalizedInbounds))]
+	appStateBz, err := cdc.MarshalJSON(&appStateGenState)
 	if err != nil {
-		return fmt.Errorf("failed to marshal zetacrosschain genesis state: %w", err)
+		return fmt.Errorf("failed to marshal crosschain genesis state: %w", err)
 	}
-	appState[crosschaintypes.ModuleName] = importedCrossChainStateBz
+	appState[crosschaintypes.ModuleName] = appStateBz
 	return nil
 }
 

cmd/zetacored/parse_genesis_test.go

@@ -153,10 +153,10 @@ func ImportGenDoc(t *testing.T, cdc *codec.ProtoCodec, n int) *types.GenesisDoc
 }
 
 func GetImportData(t *testing.T, cdc *codec.ProtoCodec, n int) map[string]json.RawMessage {
-	importData := sample.AppState(t)
+	importData := sample.AppStateLegacy(t)
 
 	// Add crosschain data to genesis state
-	importedCrossChainGenState := crosschaintypes.GetGenesisStateFromAppState(cdc, importData)
+	importedCrossChainGenState := crosschaintypes.GetGenesisStateFromAppStateLegacy(cdc, importData)
 	cctxList := make([]*crosschaintypes.CrossChainTx, n)
 	intxHashToCctxList := make([]crosschaintypes.InTxHashToCctx, n)
 	finalLizedInbounds := make([]string, n)

cmd/zetae2e/local/local.go

@@ -30,6 +30,7 @@ const (
 	flagSetupOnly         = "setup-only"
 	flagSkipSetup         = "skip-setup"
 	flagSkipBitcoinSetup  = "skip-bitcoin-setup"
+	flagSkipHeaderProof   = "skip-header-proof"
 )
 
 var (
@@ -57,6 +58,7 @@ func NewLocalCmd() *cobra.Command {
 	cmd.Flags().String(flagConfigOut, "", "config file to write the deployed contracts from the setup")
 	cmd.Flags().Bool(flagSkipSetup, false, "set to true to skip setup")
 	cmd.Flags().Bool(flagSkipBitcoinSetup, false, "set to true to skip bitcoin wallet setup")
+	cmd.Flags().Bool(flagSkipHeaderProof, false, "set to true to skip header proof tests")
 
 	return cmd
 }
@@ -111,6 +113,10 @@ func localE2ETest(cmd *cobra.Command, _ []string) {
 	if err != nil {
 		panic(err)
 	}
+	skipHeaderProof, err := cmd.Flags().GetBool(flagSkipHeaderProof)
+	if err != nil {
+		panic(err)
+	}
 
 	logger := runner.NewLogger(verbose, color.FgWhite, "setup")
 
@@ -189,8 +195,10 @@ func localE2ETest(cmd *cobra.Command, _ []string) {
 		logger.Print("⚙️ setting up networks")
 		startTime := time.Now()
 
-		if err := deployerRunner.EnableVerificationFlags(); err != nil {
-			panic(err)
+		if !skipHeaderProof {
+			if err := deployerRunner.EnableVerificationFlags(); err != nil {
+				panic(err)
+			}
 		}
 
 		deployerRunner.SetupEVM(contractsDeployed, true)
@@ -262,15 +270,15 @@ func localE2ETest(cmd *cobra.Command, _ []string) {
 		}
 		bitcoinTests := []string{
 			e2etests.TestBitcoinWithdrawSegWitName,
-			e2etests.TestBitcoinWithdrawTaprootName,
-			e2etests.TestBitcoinWithdrawLegacyName,
-			e2etests.TestBitcoinWithdrawP2SHName,
-			e2etests.TestBitcoinWithdrawP2WSHName,
 			e2etests.TestBitcoinWithdrawInvalidAddressName,
 			e2etests.TestZetaWithdrawBTCRevertName,
 			e2etests.TestCrosschainSwapName,
 		}
 		bitcoinAdvancedTests := []string{
+			e2etests.TestBitcoinWithdrawTaprootName,
+			e2etests.TestBitcoinWithdrawLegacyName,
+			e2etests.TestBitcoinWithdrawP2SHName,
+			e2etests.TestBitcoinWithdrawP2WSHName,
 			e2etests.TestBitcoinWithdrawRestrictedName,
 		}
 		ethereumTests := []string{
@@ -290,10 +298,13 @@ func localE2ETest(cmd *cobra.Command, _ []string) {
 			ethereumTests = append(ethereumTests, ethereumAdvancedTests...)
 		}
 
+		// skip the header proof test if we run light test or skipHeaderProof is enabled
+		testHeader := !light && !skipHeaderProof
+
 		eg.Go(erc20TestRoutine(conf, deployerRunner, verbose, erc20Tests...))
 		eg.Go(zetaTestRoutine(conf, deployerRunner, verbose, zetaTests...))
-		eg.Go(bitcoinTestRoutine(conf, deployerRunner, verbose, !skipBitcoinSetup, !light, bitcoinTests...))
-		eg.Go(ethereumTestRoutine(conf, deployerRunner, verbose, !light, ethereumTests...))
+		eg.Go(bitcoinTestRoutine(conf, deployerRunner, verbose, !skipBitcoinSetup, testHeader, bitcoinTests...))
+		eg.Go(ethereumTestRoutine(conf, deployerRunner, verbose, testHeader, ethereumTests...))
 	}
 	if testAdmin {
 		eg.Go(adminTestRoutine(conf, deployerRunner, verbose,

contrib/localnet/docker-compose-import-data.yml

@@ -0,0 +1,145 @@
+version: "3"
+
+# This docker-compose file configures the localnet environment
+# it contains the following services:
+# - ZetaChain with 2 nodes (zetacore0, zetacore1)
+# - A observer set with 2 clients (zetaclient0, zetaclient1)
+# - An Ethereum node (eth)
+# - A Bitcoin node (bitcoin)
+# - A Rosetta API (rosetta)
+# - An orchestrator to manage interaction with the localnet (orchestrator)
+
+networks:
+  mynetwork:
+    ipam:
+      config:
+        - subnet: 172.20.0.0/24
+
+services:
+  rosetta:
+    image: zetanode:latest
+    container_name: rosetta
+    hostname: rosetta
+    ports:
+      - "8080:8080"
+    networks:
+      mynetwork:
+        ipv4_address: 172.20.0.200
+    entrypoint: ["zetacored", "rosetta", "--tendermint", "zetacore0:26657", "--grpc", "zetacore0:9090", "--network", "athens_101-1", "--blockchain",  "zetacore" ,"--retries", "500"]
+
+  zetacore0:
+    image: zetanode:latest
+    container_name: zetacore0
+    build:
+      context: ..
+      dockerfile: Dockerfile-localnet
+    hostname: zetacore0
+    ports:
+      - "1317:1317"
+      - "9545:8545"
+      - "9546:8546"
+      - "26657:26657"
+      - "6060:6060"
+      - "9090:9090"
+    networks:
+      mynetwork:
+        ipv4_address: 172.20.0.11
+    entrypoint: ["/root/start-zetacored.sh", "2","import-data"]
+    environment:
+      - HOTKEY_BACKEND=file
+      - HOTKEY_PASSWORD=password # test purposes only
+    volumes:
+      - ~/genesis_export/:/root/genesis_data
+
+  zetacore1:
+    image: zetanode:latest
+    container_name: zetacore1
+    build:
+      context: ..
+      dockerfile: Dockerfile-localnet
+    hostname: zetacore1
+    networks:
+      mynetwork:
+        ipv4_address: 172.20.0.12
+    entrypoint: ["/root/start-zetacored.sh", "2","import-data"]
+    environment:
+      - HOTKEY_BACKEND=file
+      - HOTKEY_PASSWORD=password # test purposes only
+
+
+  zetaclient0:
+    image: zetanode:latest
+    container_name: zetaclient0
+    build:
+      context: ..
+      dockerfile: Dockerfile-localnet
+    hostname: zetaclient0
+    networks:
+      mynetwork:
+        ipv4_address: 172.20.0.21
+    entrypoint: ["/root/start-zetaclientd.sh", "import-data"]
+    environment:
+      - ETHDEV_ENDPOINT=http://eth:8545
+      - HOTKEY_BACKEND=file
+      - HOTKEY_PASSWORD=password # test purposes only
+
+  zetaclient1:
+    image: zetanode:latest
+    container_name: zetaclient1
+    build:
+      context: ..
+      dockerfile: Dockerfile-localnet
+    hostname: zetaclient1
+    networks:
+      mynetwork:
+        ipv4_address: 172.20.0.22
+    entrypoint: ["/root/start-zetaclientd.sh", "import-data"]
+    environment:
+      - ETHDEV_ENDPOINT=http://eth:8545
+      - HOTKEY_BACKEND=file
+      - HOTKEY_PASSWORD=password # test purposes only
+
+  eth:
+    image: ethereum/client-go:v1.10.26
+    container_name: eth
+    hostname: eth
+    ports:
+      - "8545:8545"
+    networks:
+      mynetwork:
+        ipv4_address: 172.20.0.100
+    entrypoint: ["geth", "--dev", "--http", "--http.addr", "172.20.0.100", "--http.vhosts", "*", "--http.api", "eth,web3,net", "--http.corsdomain", "https://remix.ethereum.org", "--dev.period", "2"]
+
+  bitcoin:
+    image: ruimarinho/bitcoin-core:22 # version 23 is not working with btcd 0.22.0 due to change in createwallet rpc
+    container_name: bitcoin
+    hostname: bitcoin
+    networks:
+      mynetwork:
+        ipv4_address: 172.20.0.101
+    ports:
+      - "18443:18443"
+    command:
+      -printtoconsole
+      -regtest=1
+      -rpcallowip=0.0.0.0/0
+      -rpcbind=0.0.0.0
+      -rpcauth=smoketest:63acf9b8dccecce914d85ff8c044b78b$$5892f9bbc84f4364e79f0970039f88bdd823f168d4acc76099ab97b14a766a99
+      -txindex=1
+
+  orchestrator:
+    image: orchestrator:latest
+    tty: true
+    container_name: orchestrator
+    build:
+      context: ..
+      dockerfile: contrib/localnet/orchestrator/Dockerfile
+    depends_on:
+      - zetacore0
+      - eth
+    hostname: orchestrator
+    networks:
+      mynetwork:
+        ipv4_address: 172.20.0.2
+    entrypoint: ["/work/start-zetae2e.sh", "local","import-data"]
+

contrib/localnet/docker-compose.yml

@@ -64,6 +64,7 @@ services:
       - HOTKEY_BACKEND=file
       - HOTKEY_PASSWORD=password # test purposes only
 
+
   zetaclient0:
     image: zetanode:latest
     container_name: zetaclient0

contrib/localnet/orchestrator/start-zetae2e.sh

@@ -9,7 +9,11 @@ ZETAE2E_CMD=$1
 OPTION=$2
 
 echo "waiting for geth RPC to start..."
-sleep 2
+ if [ "$OPTION" != "import-data" ]; then
+    sleep 2
+  else
+    sleep 520
+  fi
 
 ### Create the accounts and fund them with Ether on local Ethereum network
 
@@ -57,12 +61,12 @@ if [ "$OPTION" == "upgrade" ]; then
   # Run zetae2e, if the upgrade height is lower than 100, we use the setup-only flag
   if [ "$UPGRADE_HEIGHT" -lt 100 ]; then
     echo "running E2E command to setup the networks..."
-    zetae2e "$ZETAE2E_CMD" --setup-only --config-out deployed.yml
+    zetae2e "$ZETAE2E_CMD" --setup-only --config-out deployed.yml --skip-header-proof
   else
     echo "running E2E command to setup the networks and populate the state..."
 
     # Use light flag to ensure tests can complete before the upgrade height
-    zetae2e "$ZETAE2E_CMD" --config-out deployed.yml --light
+    zetae2e "$ZETAE2E_CMD" --config-out deployed.yml --light --skip-header-proof
  fi
   ZETAE2E_EXIT_CODE=$?
 
@@ -86,9 +90,9 @@ if [ "$OPTION" == "upgrade" ]; then
   # When the upgrade height is greater than 100 for upgrade test, the Bitcoin tests have been run once, therefore the Bitcoin wallet is already set up
   # Use light flag to skip advanced tests
   if [ "$UPGRADE_HEIGHT" -lt 100 ]; then
-    zetae2e "$ZETAE2E_CMD" --skip-setup --config deployed.yml --light
+    zetae2e "$ZETAE2E_CMD" --skip-setup --config deployed.yml --light --skip-header-proof
   else
-    zetae2e "$ZETAE2E_CMD" --skip-setup --config deployed.yml --skip-bitcoin-setup --light
+    zetae2e "$ZETAE2E_CMD" --skip-setup --config deployed.yml --skip-bitcoin-setup --light --skip-header-proof
   fi
 
   ZETAE2E_EXIT_CODE=$?

contrib/localnet/scripts/import-data.sh

@@ -0,0 +1,14 @@
+#!/bin/bash
+if [ $# -lt 1 ]
+then
+  echo "Usage: import-data.sh [network]"
+  exit 1
+fi
+
+NETWORK=$1
+echo "NETWORK: ${NETWORK}"
+mkdir ~/genesis_export/
+echo "Download Latest State Export"
+LATEST_EXPORT_URL=$(curl https://snapshots.zetachain.com/latest-state-export | jq -r ."${NETWORK}")
+echo "LATEST EXPORT URL: ${LATEST_EXPORT_URL}"
+wget -q ${LATEST_EXPORT_URL} -O ~/genesis_export/exported-genesis.json