test: add import data to e2e tests

.github/workflows/publish-release.yml

@@ -12,13 +12,28 @@ on:
         required: false
         default: false
         description: 'Use this to skip: gosec, gosec-cosmos, check-changelog, check-upgrade-uandler-updated, build-test, smoke-test and go straight to approval step.'
+      skip_release:
+        type: boolean
+        required: false
+        default: false
+        description: 'If this is true it will simply execute all the steps for a release prior to actually cutting the release, then stop'
 
 concurrency:
   group: publish-release
   cancel-in-progress: false
 
 jobs:
+  check_branch:
+    if: ${{ (startsWith(github.ref, 'refs/heads/release/v') || startsWith(github.ref, 'refs/heads/hotfix/v')) }}
+    runs-on: ubuntu-latest
+    steps:
+      - name: Branch
+        run: |
+          echo "${{ github.ref }}"
+
   gosec:
+    needs:
+      - check_branch
     runs-on: ubuntu-latest
     env:
       GO111MODULE: on
@@ -41,13 +56,15 @@ jobs:
         with:
           args: ./...
 
-      - name: Skip Checks Succeed for Needs.
+      - name: Mark Job Complete Skipped
         if: ${{ github.event.inputs.skip_checks == 'true' }}
+        shell: bash
         run: |
-          echo "Check Skipped, Mark Green for Pipeline Execution"
-
+          echo "continue"
 
   gosec-cosmos:
+    needs:
+      - check_branch
     runs-on: ubuntu-latest
     env:
       GO111MODULE: on
@@ -68,12 +85,15 @@ jobs:
         if: ${{ github.event.inputs.skip_checks != 'true' }}
         run: make lint-cosmos-gosec
 
-      - name: Skip Checks Succeed for Needs.
+      - name: Mark Job Complete Skipped
         if: ${{ github.event.inputs.skip_checks == 'true' }}
+        shell: bash
         run: |
-          echo "Check Skipped, Mark Green for Pipeline Execution"
+          echo "continue"
 
   lint:
+    needs:
+      - check_branch
     runs-on: ubuntu-latest
     timeout-minutes: 15
     env:
@@ -99,12 +119,15 @@ jobs:
           skip-cache: true
           args: --out-format=json
 
-      - name: Skip Checks Succeed for Needs.
+      - name: Mark Job Complete Skipped
         if: ${{ github.event.inputs.skip_checks == 'true' }}
+        shell: bash
         run: |
-          echo "Check Skipped, Mark Green for Pipeline Execution"
+          echo "continue"
 
   check-changelog:
+    needs:
+      - check_branch
     runs-on: ubuntu-latest
     steps:
 
@@ -122,7 +145,6 @@ jobs:
           https://api.github.com/repos/${{ github.repository }}/git/ref/heads/develop | jq -r '.object.sha')
           echo "DEVELOP_SHA=${SHA}" >> ${GITHUB_ENV}
           echo "CURRENT_BRANCH_SHA=${{ github.sha }}" >> ${GITHUB_ENV}
-
 
       - name: Check for CHANGELOG.md changes
         if: ${{ github.event.inputs.skip_checks != 'true' }}
@@ -139,12 +161,15 @@ jobs:
             echo "CHANGELOG.md has been updated."
           fi
 
-      - name: Skip Checks Succeed for Needs.
+      - name: Mark Job Complete Skipped
         if: ${{ github.event.inputs.skip_checks == 'true' }}
+        shell: bash
         run: |
-          echo "Check Skipped, Mark Green for Pipeline Execution"
+          echo "continue"
 
   check-upgrade-handler-updated:
+    needs:
+      - check_branch
     runs-on: ubuntu-latest
     timeout-minutes: 10
     steps:
@@ -168,12 +193,15 @@ jobs:
           fi
           echo "The major version found in 'releaseVersion' in app/setup_handlers.go matches this tagged release - Moving Forward!"
 
-      - name: Skip Checks Succeed for Needs.
+      - name: Mark Job Complete Skipped
         if: ${{ github.event.inputs.skip_checks == 'true' }}
+        shell: bash
         run: |
-          echo "Check Skipped, Mark Green for Pipeline Execution"
+          echo "continue"
 
   build-test:
+    needs:
+      - check_branch
     runs-on: buildjet-4vcpu-ubuntu-2004
     timeout-minutes: 15
     concurrency:
@@ -242,12 +270,15 @@ jobs:
         shell: bash
         run: rm -rf *
 
-      - name: Skip Checks Succeed for Needs.
+      - name: Mark Job Complete Skipped
         if: ${{ github.event.inputs.skip_checks == 'true' }}
+        shell: bash
         run: |
-          echo "Check Skipped, Mark Green for Pipeline Execution"
+          echo "continue"
 
   smoke-test:
+    needs:
+      - check_branch
     runs-on: buildjet-4vcpu-ubuntu-2204
     timeout-minutes: 25
     steps:
@@ -317,15 +348,15 @@ jobs:
         shell: bash
         run: sudo rm -rf *
 
-      - name: Skip Checks Succeed for Needs.
+      - name: Mark Job Complete Skipped
         if: ${{ github.event.inputs.skip_checks == 'true' }}
+        shell: bash
         run: |
-          echo "Check Skipped, Mark Green for Pipeline Execution"
-
-
-
+          echo "continue"
 
   e2e-admin-tests:
+    needs:
+      - check_branch
     runs-on: buildjet-4vcpu-ubuntu-2204
     timeout-minutes: 120
     steps:
@@ -339,7 +370,15 @@ jobs:
         run: |
           make start-e2e-admin-test
 
+      - name: Mark Job Complete Skipped
+        if: ${{ github.event.inputs.skip_checks == 'true' }}
+        shell: bash
+        run: |
+          echo "continue"
+
   e2e-upgrade-test:
+    needs:
+      - check_branch
     runs-on: buildjet-4vcpu-ubuntu-2204
     timeout-minutes: 120
     steps:
@@ -351,11 +390,16 @@ jobs:
         if: ${{ github.event.inputs.skip_checks != 'true' }}
         shell: bash
         run: |
-          make start-upgrade-test
+          make start-e2e-import-mainnet-test
 
-  approval:
-    runs-on: ubuntu-latest
-    timeout-minutes: 10
+      - name: Mark Job Complete Skipped
+        if: ${{ github.event.inputs.skip_checks == 'true' }}
+        shell: bash
+        run: |
+          echo "continue"
+
+  publish-release:
+    if: ${{ github.event.inputs.skip_release == 'false' }}
     needs:
       - gosec
       - gosec-cosmos
@@ -366,24 +410,10 @@ jobs:
       - build-test
       - e2e-admin-tests
       - e2e-upgrade-test
-    steps:
-
-      - uses: trstringer/manual-approval@v1
-        with:
-          secret: ${{ github.TOKEN }}
-          approvers: kingpinXD,lumtis,brewmaster012,CharlieMc0
-          minimum-approvals: 1
-          issue-title: "[Release] Approval, version: ${{ github.event.inputs.version }}"
-          issue-body: "Once approved the release pipeline will continue."
-          exclude-workflow-initiator-as-approver: true
-          additional-approved-words: ''
-          additional-denied-words: ''
-  publish-release:
+      - check_branch
     runs-on: buildjet-4vcpu-ubuntu-2004
     timeout-minutes: 60
-    needs:
-      - approval
-
+    environment: release
     steps:
       - uses: actions/checkout@v3
 

Dockerfile-upgrade

@@ -20,8 +20,8 @@ WORKDIR /go/delivery/zeta-node
 RUN mkdir -p  $GOPATH/bin/old
 RUN mkdir -p  $GOPATH/bin/new
 
-ARG OLD_VERSION=v14.0.0
-ENV NEW_VERSION=v15
+ARG OLD_VERSION=v15.0.0
+ENV NEW_VERSION=v16
 
 # Build new release from the current source
 COPY go.mod /go/delivery/zeta-node/

Makefile

@@ -206,10 +206,15 @@ start-e2e-admin-test: zetanode
 	@echo "--> Starting e2e admin test"
 	cd contrib/localnet/ && $(DOCKER) compose -f docker-compose.yml -f docker-compose-admin.yml up -d
 
+start-e2e-import-mainnet-test: zetanode
+	@echo "--> Starting e2e import-data test"
+	cd contrib/localnet/  && chmod +x scripts/import-data.sh && ./scripts/import-data.sh mainnet && $(DOCKER) compose -f docker-compose.yml -f docker-compose-import-data.yml up -d
+
 start-e2e-performance-test: zetanode
 	@echo "--> Starting e2e performance test"
 	cd contrib/localnet/ && $(DOCKER) compose -f docker-compose.yml -f docker-compose-performance.yml up -d
 
+
 start-stress-test: zetanode
 	@echo "--> Starting stress test"
 	cd contrib/localnet/ && $(DOCKER) compose -f docker-compose.yml -f docker-compose-stresstest.yml up -d

app/setup_handlers.go

@@ -10,7 +10,7 @@ import (
 	observertypes "github.com/zeta-chain/zetacore/x/observer/types"
 )
 
-const releaseVersion = "v15"
+const releaseVersion = "v16"
 
 func SetupHandlers(app *App) {
 	app.UpgradeKeeper.SetUpgradeHandler(releaseVersion, func(ctx sdk.Context, _ types.Plan, vm module.VersionMap) (module.VersionMap, error) {

changelog.md

@@ -83,6 +83,7 @@
 * [1985](https://github.com/zeta-chain/node/pull/1985) - improve fungible module coverage
 * [1992](https://github.com/zeta-chain/node/pull/1992) - remove setupKeeper from crosschain module
 * [2008](https://github.com/zeta-chain/node/pull/2008) - add test for connector bytecode update
+* [2127](https://github.com/zeta-chain/node/pull/2127) - add import data to e2e test
 
 ### Fixes
 

cmd/zetacored/parse_genesis.go

@@ -47,20 +47,30 @@ var Copy = map[string]bool{
 	upgradetypes.ModuleName:   true,
 	evidencetypes.ModuleName:  true,
 	vestingtypes.ModuleName:   true,
-	fungibletypes.ModuleName:  true,
 	emissionstypes.ModuleName: true,
-	authz.ModuleName:          true,
 }
 
 // Skip represents a set of modules for which, the entire state is skipped and nothing gets imported
 var Skip = map[string]bool{
-	evmtypes.ModuleName:          true,
-	stakingtypes.ModuleName:      true,
-	genutiltypes.ModuleName:      true,
-	authtypes.ModuleName:         true,
-	banktypes.ModuleName:         true,
+	// Skipping evm this is done to reduce the size of the genesis file evm module uses the majority of the space due to smart contract data
+	evmtypes.ModuleName: true,
+	// Skipping staking as new validators would be created for the new chain
+	stakingtypes.ModuleName: true,
+	// Skipping genutil as new gentxs would be created
+	genutiltypes.ModuleName: true,
+	// Skipping auth as new accounts would be created for the new chain. This also needs to be done as we are skipping evm module
+	authtypes.ModuleName: true,
+	// Skipping bank module as it is not used when starting a new chain this is done to make sure the total supply invariant is maintained.
+	// This would need modification but might be possible to add in non evm based modules in the future
+	banktypes.ModuleName: true,
+	// Skipping distribution module as it is not used when starting a new chain , rewards are based on validators and delegators , and so rewards from a different chain do not hold any value
 	distributiontypes.ModuleName: true,
-	group.ModuleName:             true,
+	// Skipping group module as it is not used when starting a new chain, new groups should be created based on the validator operator keys
+	group.ModuleName: true,
+	// Skipping authz as it is not used when starting a new chain, new grants should be created based on the validator hotkeys abd operator keys
+	authz.ModuleName: true,
+	// Skipping fungible module as new fungible tokens would be created and system contract would be deployed
+	fungibletypes.ModuleName: true,
 }
 
 // Modify represents a set of modules for which, the state is modified before importing. Each Module should have a corresponding Modify function
@@ -140,7 +150,6 @@ func ImportDataIntoFile(genDoc *types.GenesisDoc, importFile *types.GenesisDoc,
 			}
 		}
 	}
-
 	appStateJSON, err := json.Marshal(appState)
 	if err != nil {
 		return fmt.Errorf("failed to marshal application genesis state: %w", err)
@@ -153,15 +162,18 @@ func ImportDataIntoFile(genDoc *types.GenesisDoc, importFile *types.GenesisDoc,
 // ModifyCrosschainState modifies the crosschain state before importing
 // It truncates the crosschain transactions, inbound transactions and finalized inbounds to MaxItemsForList
 func ModifyCrosschainState(appState map[string]json.RawMessage, importAppState map[string]json.RawMessage, cdc codec.Codec) error {
-	importedCrossChainGenState := crosschaintypes.GetGenesisStateFromAppState(cdc, importAppState)
-	importedCrossChainGenState.CrossChainTxs = importedCrossChainGenState.CrossChainTxs[:math.Min(MaxItemsForList, len(importedCrossChainGenState.CrossChainTxs))]
-	importedCrossChainGenState.InTxHashToCctxList = importedCrossChainGenState.InTxHashToCctxList[:math.Min(MaxItemsForList, len(importedCrossChainGenState.InTxHashToCctxList))]
-	importedCrossChainGenState.FinalizedInbounds = importedCrossChainGenState.FinalizedInbounds[:math.Min(MaxItemsForList, len(importedCrossChainGenState.FinalizedInbounds))]
-	importedCrossChainStateBz, err := json.Marshal(importedCrossChainGenState)
+	importedCrossChainGenState := crosschaintypes.GetGenesisStateFromAppStateLegacy(cdc, importAppState)
+	appStateGenState := crosschaintypes.GetGenesisStateFromAppState(cdc, appState)
+	// The genesis state has been modified between the two versions, so we add only the required fields and leave out the rest
+	// v16 adds the rate_limiter_flags and removes params from the genesis state
+	appStateGenState.CrossChainTxs = importedCrossChainGenState.CrossChainTxs[:math.Min(MaxItemsForList, len(importedCrossChainGenState.CrossChainTxs))]
+	appStateGenState.InTxHashToCctxList = importedCrossChainGenState.InTxHashToCctxList[:math.Min(MaxItemsForList, len(importedCrossChainGenState.InTxHashToCctxList))]
+	appStateGenState.FinalizedInbounds = importedCrossChainGenState.FinalizedInbounds[:math.Min(MaxItemsForList, len(importedCrossChainGenState.FinalizedInbounds))]
+	appStateBz, err := cdc.MarshalJSON(&appStateGenState)
 	if err != nil {
-		return fmt.Errorf("failed to marshal zetacrosschain genesis state: %w", err)
+		return fmt.Errorf("failed to marshal crosschain genesis state: %w", err)
 	}
-	appState[crosschaintypes.ModuleName] = importedCrossChainStateBz
+	appState[crosschaintypes.ModuleName] = appStateBz
 	return nil
 }
 

cmd/zetacored/parse_genesis_test.go

@@ -153,10 +153,10 @@ func ImportGenDoc(t *testing.T, cdc *codec.ProtoCodec, n int) *types.GenesisDoc
 }
 
 func GetImportData(t *testing.T, cdc *codec.ProtoCodec, n int) map[string]json.RawMessage {
-	importData := sample.AppState(t)
+	importData := sample.AppStateLegacy(t)
 
 	// Add crosschain data to genesis state
-	importedCrossChainGenState := crosschaintypes.GetGenesisStateFromAppState(cdc, importData)
+	importedCrossChainGenState := crosschaintypes.GetGenesisStateFromAppStateLegacy(cdc, importData)
 	cctxList := make([]*crosschaintypes.CrossChainTx, n)
 	intxHashToCctxList := make([]crosschaintypes.InTxHashToCctx, n)
 	finalLizedInbounds := make([]string, n)