test: add import data to e2e tests

.github/workflows/publish-release.yml

@@ -398,6 +398,30 @@ jobs:
         run: |
           echo "continue"
 
+  e2e-stateful-data-test:
+    needs:
+      - check_branch
+    runs-on: buildjet-4vcpu-ubuntu-2204
+    timeout-minutes: 120
+    steps:
+      - name: "Checkout Code"
+        if: ${{ github.event.inputs.skip_checks != 'true' }}
+        uses: actions/checkout@v3
+
+      - name: Execute stateful-data-test
+        if: ${{ github.event.inputs.skip_checks != 'true' }}
+        shell: bash
+        run: |
+          make start-e2e-import-mainnet-test
+          container_id=$(docker ps --filter "ancestor=orchestrator:latest" --format "{{.ID}}")
+          docker logs -f "${container_id}" & exit $(docker wait "${container_id}")
+
+      - name: Mark Job Complete Skipped
+        if: ${{ github.event.inputs.skip_checks == 'true' }}
+        shell: bash
+        run: |
+          echo "continue"
+
   publish-release:
     if: ${{ github.event.inputs.skip_release == 'false' }}
     needs:
@@ -411,6 +435,7 @@ jobs:
       - e2e-admin-tests
       - e2e-upgrade-test
       - check_branch
+      - e2e-stateful-data-test
     runs-on: buildjet-4vcpu-ubuntu-2004
     timeout-minutes: 60
     environment: release

Makefile

@@ -206,10 +206,15 @@ start-e2e-admin-test: zetanode
 	@echo "--> Starting e2e admin test"
 	cd contrib/localnet/ && $(DOCKER) compose -f docker-compose.yml -f docker-compose-admin.yml up -d
 
+start-e2e-import-mainnet-test: zetanode
+	@echo "--> Starting e2e import-data test"
+	cd contrib/localnet/  && ./scripts/import-data.sh mainnet && $(DOCKER) compose -f docker-compose.yml -f docker-compose-import-data.yml up -d
+
 start-e2e-performance-test: zetanode
 	@echo "--> Starting e2e performance test"
 	cd contrib/localnet/ && $(DOCKER) compose -f docker-compose.yml -f docker-compose-performance.yml up -d
 
+
 start-stress-test: zetanode
 	@echo "--> Starting stress test"
 	cd contrib/localnet/ && $(DOCKER) compose -f docker-compose.yml -f docker-compose-stresstest.yml up -d

changelog.md

@@ -85,6 +85,7 @@
 * [1992](https://github.com/zeta-chain/node/pull/1992) - remove setupKeeper from crosschain module
 * [2008](https://github.com/zeta-chain/node/pull/2008) - add test for connector bytecode update
 * [2060](https://github.com/zeta-chain/node/pull/2060) - add unit test for rate limiter query
+* [2127](https://github.com/zeta-chain/node/pull/2127) - add import data to e2e test
 
 ### Fixes
 

cmd/zetacored/parse_genesis.go

@@ -47,20 +47,30 @@ var Copy = map[string]bool{
 	upgradetypes.ModuleName:   true,
 	evidencetypes.ModuleName:  true,
 	vestingtypes.ModuleName:   true,
-	fungibletypes.ModuleName:  true,
 	emissionstypes.ModuleName: true,
-	authz.ModuleName:          true,
 }
 
 // Skip represents a set of modules for which, the entire state is skipped and nothing gets imported
 var Skip = map[string]bool{
-	evmtypes.ModuleName:          true,
-	stakingtypes.ModuleName:      true,
-	genutiltypes.ModuleName:      true,
-	authtypes.ModuleName:         true,
-	banktypes.ModuleName:         true,
+	// Skipping evm this is done to reduce the size of the genesis file evm module uses the majority of the space due to smart contract data
+	evmtypes.ModuleName: true,
+	// Skipping staking as new validators would be created for the new chain
+	stakingtypes.ModuleName: true,
+	// Skipping genutil as new gentxs would be created
+	genutiltypes.ModuleName: true,
+	// Skipping auth as new accounts would be created for the new chain. This also needs to be done as we are skipping evm module
+	authtypes.ModuleName: true,
+	// Skipping bank module as it is not used when starting a new chain this is done to make sure the total supply invariant is maintained.
+	// This would need modification but might be possible to add in non evm based modules in the future
+	banktypes.ModuleName: true,
+	// Skipping distribution module as it is not used when starting a new chain , rewards are based on validators and delegators , and so rewards from a different chain do not hold any value
 	distributiontypes.ModuleName: true,
-	group.ModuleName:             true,
+	// Skipping group module as it is not used when starting a new chain, new groups should be created based on the validator operator keys
+	group.ModuleName: true,
+	// Skipping authz as it is not used when starting a new chain, new grants should be created based on the validator hotkeys abd operator keys
+	authz.ModuleName: true,
+	// Skipping fungible module as new fungible tokens would be created and system contract would be deployed
+	fungibletypes.ModuleName: true,
 }
 
 // Modify represents a set of modules for which, the state is modified before importing. Each Module should have a corresponding Modify function
@@ -140,7 +150,6 @@ func ImportDataIntoFile(genDoc *types.GenesisDoc, importFile *types.GenesisDoc,
 			}
 		}
 	}
-
 	appStateJSON, err := json.Marshal(appState)
 	if err != nil {
 		return fmt.Errorf("failed to marshal application genesis state: %w", err)
@@ -153,15 +162,18 @@ func ImportDataIntoFile(genDoc *types.GenesisDoc, importFile *types.GenesisDoc,
 // ModifyCrosschainState modifies the crosschain state before importing
 // It truncates the crosschain transactions, inbound transactions and finalized inbounds to MaxItemsForList
 func ModifyCrosschainState(appState map[string]json.RawMessage, importAppState map[string]json.RawMessage, cdc codec.Codec) error {
-	importedCrossChainGenState := crosschaintypes.GetGenesisStateFromAppState(cdc, importAppState)
-	importedCrossChainGenState.CrossChainTxs = importedCrossChainGenState.CrossChainTxs[:math.Min(MaxItemsForList, len(importedCrossChainGenState.CrossChainTxs))]
-	importedCrossChainGenState.InTxHashToCctxList = importedCrossChainGenState.InTxHashToCctxList[:math.Min(MaxItemsForList, len(importedCrossChainGenState.InTxHashToCctxList))]
-	importedCrossChainGenState.FinalizedInbounds = importedCrossChainGenState.FinalizedInbounds[:math.Min(MaxItemsForList, len(importedCrossChainGenState.FinalizedInbounds))]
-	importedCrossChainStateBz, err := json.Marshal(importedCrossChainGenState)
+	importedCrossChainGenState := crosschaintypes.GetGenesisStateFromAppStateLegacy(cdc, importAppState)
+	appStateGenState := crosschaintypes.GetGenesisStateFromAppState(cdc, appState)
+	// The genesis state has been modified between the two versions, so we add only the required fields and leave out the rest
+	// v16 adds the rate_limiter_flags and removes params from the genesis state
+	appStateGenState.CrossChainTxs = importedCrossChainGenState.CrossChainTxs[:math.Min(MaxItemsForList, len(importedCrossChainGenState.CrossChainTxs))]
+	appStateGenState.InTxHashToCctxList = importedCrossChainGenState.InTxHashToCctxList[:math.Min(MaxItemsForList, len(importedCrossChainGenState.InTxHashToCctxList))]
+	appStateGenState.FinalizedInbounds = importedCrossChainGenState.FinalizedInbounds[:math.Min(MaxItemsForList, len(importedCrossChainGenState.FinalizedInbounds))]
+	appStateBz, err := cdc.MarshalJSON(&appStateGenState)
 	if err != nil {
-		return fmt.Errorf("failed to marshal zetacrosschain genesis state: %w", err)
+		return fmt.Errorf("failed to marshal crosschain genesis state: %w", err)
 	}
-	appState[crosschaintypes.ModuleName] = importedCrossChainStateBz
+	appState[crosschaintypes.ModuleName] = appStateBz
 	return nil
 }
 

cmd/zetacored/parse_genesis_test.go

@@ -153,10 +153,10 @@ func ImportGenDoc(t *testing.T, cdc *codec.ProtoCodec, n int) *types.GenesisDoc
 }
 
 func GetImportData(t *testing.T, cdc *codec.ProtoCodec, n int) map[string]json.RawMessage {
-	importData := sample.AppState(t)
+	importData := sample.AppStateLegacy(t)
 
 	// Add crosschain data to genesis state
-	importedCrossChainGenState := crosschaintypes.GetGenesisStateFromAppState(cdc, importData)
+	importedCrossChainGenState := crosschaintypes.GetGenesisStateFromAppStateLegacy(cdc, importData)
 	cctxList := make([]*crosschaintypes.CrossChainTx, n)
 	intxHashToCctxList := make([]crosschaintypes.InTxHashToCctx, n)
 	finalLizedInbounds := make([]string, n)

contrib/localnet/docker-compose-import-data.yml

@@ -0,0 +1,145 @@
+version: "3"
+
+# This docker-compose file configures the localnet environment
+# it contains the following services:
+# - ZetaChain with 2 nodes (zetacore0, zetacore1)
+# - A observer set with 2 clients (zetaclient0, zetaclient1)
+# - An Ethereum node (eth)
+# - A Bitcoin node (bitcoin)
+# - A Rosetta API (rosetta)
+# - An orchestrator to manage interaction with the localnet (orchestrator)
+
+networks:
+  mynetwork:
+    ipam:
+      config:
+        - subnet: 172.20.0.0/24
+
+services:
+  rosetta:
+    image: zetanode:latest
+    container_name: rosetta
+    hostname: rosetta
+    ports:
+      - "8080:8080"
+    networks:
+      mynetwork:
+        ipv4_address: 172.20.0.200
+    entrypoint: ["zetacored", "rosetta", "--tendermint", "zetacore0:26657", "--grpc", "zetacore0:9090", "--network", "athens_101-1", "--blockchain",  "zetacore" ,"--retries", "500"]
+
+  zetacore0:
+    image: zetanode:latest
+    container_name: zetacore0
+    build:
+      context: ..
+      dockerfile: Dockerfile-localnet
+    hostname: zetacore0
+    ports:
+      - "1317:1317"
+      - "9545:8545"
+      - "9546:8546"
+      - "26657:26657"
+      - "6060:6060"
+      - "9090:9090"
+    networks:
+      mynetwork:
+        ipv4_address: 172.20.0.11
+    entrypoint: ["/root/start-zetacored.sh", "2","import-data"]
+    environment:
+      - HOTKEY_BACKEND=file
+      - HOTKEY_PASSWORD=password # test purposes only
+    volumes:
+      - ~/genesis_export/:/root/genesis_data
+
+  zetacore1:
+    image: zetanode:latest
+    container_name: zetacore1
+    build:
+      context: ..
+      dockerfile: Dockerfile-localnet
+    hostname: zetacore1
+    networks:
+      mynetwork:
+        ipv4_address: 172.20.0.12
+    entrypoint: ["/root/start-zetacored.sh", "2","import-data"]
+    environment:
+      - HOTKEY_BACKEND=file
+      - HOTKEY_PASSWORD=password # test purposes only
+
+
+  zetaclient0:
+    image: zetanode:latest
+    container_name: zetaclient0
+    build:
+      context: ..
+      dockerfile: Dockerfile-localnet
+    hostname: zetaclient0
+    networks:
+      mynetwork:
+        ipv4_address: 172.20.0.21
+    entrypoint: ["/root/start-zetaclientd.sh", "import-data"]
+    environment:
+      - ETHDEV_ENDPOINT=http://eth:8545
+      - HOTKEY_BACKEND=file
+      - HOTKEY_PASSWORD=password # test purposes only
+
+  zetaclient1:
+    image: zetanode:latest
+    container_name: zetaclient1
+    build:
+      context: ..
+      dockerfile: Dockerfile-localnet
+    hostname: zetaclient1
+    networks:
+      mynetwork:
+        ipv4_address: 172.20.0.22
+    entrypoint: ["/root/start-zetaclientd.sh", "import-data"]
+    environment:
+      - ETHDEV_ENDPOINT=http://eth:8545
+      - HOTKEY_BACKEND=file
+      - HOTKEY_PASSWORD=password # test purposes only
+
+  eth:
+    image: ethereum/client-go:v1.10.26
+    container_name: eth
+    hostname: eth
+    ports:
+      - "8545:8545"
+    networks:
+      mynetwork:
+        ipv4_address: 172.20.0.100
+    entrypoint: ["geth", "--dev", "--http", "--http.addr", "172.20.0.100", "--http.vhosts", "*", "--http.api", "eth,web3,net", "--http.corsdomain", "https://remix.ethereum.org", "--dev.period", "2"]
+
+  bitcoin:
+    image: ruimarinho/bitcoin-core:22 # version 23 is not working with btcd 0.22.0 due to change in createwallet rpc
+    container_name: bitcoin
+    hostname: bitcoin
+    networks:
+      mynetwork:
+        ipv4_address: 172.20.0.101
+    ports:
+      - "18443:18443"
+    command:
+      -printtoconsole
+      -regtest=1
+      -rpcallowip=0.0.0.0/0
+      -rpcbind=0.0.0.0
+      -rpcauth=smoketest:63acf9b8dccecce914d85ff8c044b78b$$5892f9bbc84f4364e79f0970039f88bdd823f168d4acc76099ab97b14a766a99
+      -txindex=1
+
+  orchestrator:
+    image: orchestrator:latest
+    tty: true
+    container_name: orchestrator
+    build:
+      context: ..
+      dockerfile: contrib/localnet/orchestrator/Dockerfile
+    depends_on:
+      - zetacore0
+      - eth
+    hostname: orchestrator
+    networks:
+      mynetwork:
+        ipv4_address: 172.20.0.2
+    entrypoint: ["/work/start-zetae2e.sh", "local","import-data"]
+

contrib/localnet/docker-compose.yml

@@ -64,6 +64,7 @@ services:
       - HOTKEY_BACKEND=file
       - HOTKEY_PASSWORD=password # test purposes only
 
+
   zetaclient0:
     image: zetanode:latest
     container_name: zetaclient0

contrib/localnet/orchestrator/start-zetae2e.sh

@@ -9,7 +9,13 @@ ZETAE2E_CMD=$1
 OPTION=$2
 
 echo "waiting for geth RPC to start..."
-sleep 2
+#  pause nodes other than zetacore0 to wait for zetacore0 to create genesis.json
+#  additional pause time is needed for importing data into the genesis as the export file is read into memory
+ if [ "$OPTION" != "import-data" ]; then
+    sleep 2
+  else
+    sleep 520
+  fi
 
 ### Create the accounts and fund them with Ether on local Ethereum network
 

contrib/localnet/scripts/import-data.sh

@@ -0,0 +1,15 @@
+#!/bin/bash
+if [ $# -lt 1 ]
+then
+  echo "Usage: import-data.sh [network]"
+  exit 1
+fi
+
+NETWORK=$1
+echo "NETWORK: ${NETWORK}"
+rm -rf ~/genesis_export/
+mkdir ~/genesis_export/
+echo "Download Latest State Export"
+LATEST_EXPORT_URL=$(curl https://snapshots.zetachain.com/latest-state-export | jq -r ."${NETWORK}")
+echo "LATEST EXPORT URL: ${LATEST_EXPORT_URL}"
+wget -q ${LATEST_EXPORT_URL} -O ~/genesis_export/exported-genesis.json

contrib/localnet/scripts/start-zetaclientd.sh

@@ -25,7 +25,14 @@ num=$(echo $HOSTNAME | tr -dc '0-9')
 node="zetacore$num"
 
 echo "Wait for zetacore to exchange genesis file"
-sleep 40
+#  pause nodes other than zetacore0 to wait for zetacore0 to create genesis.json
+#  additional pause time is needed for importing data into the genesis as the export file is read into memory
+if [ "$OPTION" != "import-data" ]; then
+  sleep 40
+else
+  sleep 510
+fi
+
 operator=$(cat $HOME/.zetacored/os.json | jq '.ObserverAddress' )
 operatorAddress=$(echo "$operator" | tr -d '"')
 echo "operatorAddress: $operatorAddress"

contrib/localnet/scripts/start-zetacored.sh

@@ -69,8 +69,15 @@ source ~/add-keys.sh
 # Pause other nodes so that the primary can node can do the genesis creation
 if [ $HOSTNAME != "zetacore0" ]
 then
+#  pause nodes other than zetacore0 to wait for zetacore0 to create genesis.json
+#  additional pause time is needed for importing data into the genesis as the export file is read into memory
+
   echo "Waiting for zetacore0 to create genesis.json"
-  sleep 10
+  if [ "$OPTION" != "import-data" ]; then
+    sleep 10
+  else
+    sleep 500
+  fi
   echo "genesis.json created"
 fi
 
@@ -155,9 +162,17 @@ then
 
 # 4. Collect all the gentx files in zetacore0 and create the final genesis.json
   zetacored collect-gentxs
+
+  if [ "$OPTION" == "import-data" ]; then
+    echo "Importing data"
+    zetacored parse-genesis-file /root/genesis_data/exported-genesis.json
+  fi
+
   zetacored validate-genesis
+
 # 5. Copy the final genesis.json to all the nodes
   for NODE in "${NODELIST[@]}"; do
+      echo "Copying genesis.json to $NODE"
       ssh $NODE rm -rf ~/.zetacored/genesis.json
       scp ~/.zetacored/config/genesis.json $NODE:~/.zetacored/config/genesis.json
   done