test: add import data to e2e tests (#2127)

E2E Testing Improvements:
Disabled the header proof test in local upgrade E2E tests.
Introduced an option to skip the header proof test.
Utilized the new skip option for migration testing.
Moved Bitcoin address tests to the advanced section.

Logging and Version:
Displayed cross-chain transaction (cctx) details in logs.
Updated the application version.

Fixes and Enhancements:
Corrected verification flags error.
Added legacy genesis structures.
Implemented E2E testing with imported data.
Updated the upgrade testing to utilize state export.

Code Maintenance:
Added latest genesis data to a new command.
Included additional comments and resolved unit test issues.
Removed unused functions.
Parameterized the import data script.
Documented a changelog and generated files.
Added an issue to track the removal of legacy modules.

Pipeline Updates:
Updated the release pipeline, including removing unnecessary chmod commands from the Makefile.

.github/workflows/publish-release.yml

@@ -398,6 +398,30 @@ jobs:
         run: |
           echo "continue"
 
+  e2e-stateful-data-test:
+    needs:
+      - check_branch
+    runs-on: buildjet-4vcpu-ubuntu-2204
+    timeout-minutes: 120
+    steps:
+      - name: "Checkout Code"
+        if: ${{ github.event.inputs.skip_checks != 'true' }}
+        uses: actions/checkout@v3
+
+      - name: Execute stateful-data-test
+        if: ${{ github.event.inputs.skip_checks != 'true' }}
+        shell: bash
+        run: |
+          make start-e2e-import-mainnet-test
+          container_id=$(docker ps --filter "ancestor=orchestrator:latest" --format "{{.ID}}")
+          docker logs -f "${container_id}" & exit $(docker wait "${container_id}")
+
+      - name: Mark Job Complete Skipped
+        if: ${{ github.event.inputs.skip_checks == 'true' }}
+        shell: bash
+        run: |
+          echo "continue"
+
   publish-release:
     if: ${{ github.event.inputs.skip_release == 'false' }}
     needs:
@@ -411,6 +435,7 @@ jobs:
       - e2e-admin-tests
       - e2e-upgrade-test
       - check_branch
+      - e2e-stateful-data-test
     runs-on: buildjet-4vcpu-ubuntu-2004
     timeout-minutes: 60
     environment: release

Makefile

@@ -206,10 +206,15 @@ start-e2e-admin-test: zetanode
 	@echo "--> Starting e2e admin test"
 	cd contrib/localnet/ && $(DOCKER) compose -f docker-compose.yml -f docker-compose-admin.yml up -d
 
+start-e2e-import-mainnet-test: zetanode
+	@echo "--> Starting e2e import-data test"
+	cd contrib/localnet/  && ./scripts/import-data.sh mainnet && $(DOCKER) compose -f docker-compose.yml -f docker-compose-import-data.yml up -d
+
 start-e2e-performance-test: zetanode
 	@echo "--> Starting e2e performance test"
 	cd contrib/localnet/ && $(DOCKER) compose -f docker-compose.yml -f docker-compose-performance.yml up -d
 
+
 start-stress-test: zetanode
 	@echo "--> Starting stress test"
 	cd contrib/localnet/ && $(DOCKER) compose -f docker-compose.yml -f docker-compose-stresstest.yml up -d

changelog.md

@@ -85,6 +85,7 @@
 * [1992](https://github.com/zeta-chain/node/pull/1992) - remove setupKeeper from crosschain module
 * [2008](https://github.com/zeta-chain/node/pull/2008) - add test for connector bytecode update
 * [2060](https://github.com/zeta-chain/node/pull/2060) - add unit test for rate limiter query
+* [2127](https://github.com/zeta-chain/node/pull/2127) - add import data to e2e test
 
 ### Fixes
 

cmd/zetacored/parse_genesis.go

@@ -47,20 +47,30 @@ var Copy = map[string]bool{
 	upgradetypes.ModuleName:   true,
 	evidencetypes.ModuleName:  true,
 	vestingtypes.ModuleName:   true,
-	fungibletypes.ModuleName:  true,
 	emissionstypes.ModuleName: true,
-	authz.ModuleName:          true,
 }
 
 // Skip represents a set of modules for which, the entire state is skipped and nothing gets imported
 var Skip = map[string]bool{
-	evmtypes.ModuleName:          true,
-	stakingtypes.ModuleName:      true,
-	genutiltypes.ModuleName:      true,
-	authtypes.ModuleName:         true,
-	banktypes.ModuleName:         true,
+	// Skipping evm this is done to reduce the size of the genesis file evm module uses the majority of the space due to smart contract data
+	evmtypes.ModuleName: true,
+	// Skipping staking as new validators would be created for the new chain
+	stakingtypes.ModuleName: true,
+	// Skipping genutil as new gentxs would be created
+	genutiltypes.ModuleName: true,
+	// Skipping auth as new accounts would be created for the new chain. This also needs to be done as we are skipping evm module
+	authtypes.ModuleName: true,
+	// Skipping bank module as it is not used when starting a new chain this is done to make sure the total supply invariant is maintained.
+	// This would need modification but might be possible to add in non evm based modules in the future
+	banktypes.ModuleName: true,
+	// Skipping distribution module as it is not used when starting a new chain , rewards are based on validators and delegators , and so rewards from a different chain do not hold any value
 	distributiontypes.ModuleName: true,
-	group.ModuleName:             true,
+	// Skipping group module as it is not used when starting a new chain, new groups should be created based on the validator operator keys
+	group.ModuleName: true,
+	// Skipping authz as it is not used when starting a new chain, new grants should be created based on the validator hotkeys abd operator keys
+	authz.ModuleName: true,
+	// Skipping fungible module as new fungible tokens would be created and system contract would be deployed
+	fungibletypes.ModuleName: true,
 }
 
 // Modify represents a set of modules for which, the state is modified before importing. Each Module should have a corresponding Modify function
@@ -140,7 +150,6 @@ func ImportDataIntoFile(genDoc *types.GenesisDoc, importFile *types.GenesisDoc,
 			}
 		}
 	}
-
 	appStateJSON, err := json.Marshal(appState)
 	if err != nil {
 		return fmt.Errorf("failed to marshal application genesis state: %w", err)
@@ -153,15 +162,18 @@ func ImportDataIntoFile(genDoc *types.GenesisDoc, importFile *types.GenesisDoc,
 // ModifyCrosschainState modifies the crosschain state before importing
 // It truncates the crosschain transactions, inbound transactions and finalized inbounds to MaxItemsForList
 func ModifyCrosschainState(appState map[string]json.RawMessage, importAppState map[string]json.RawMessage, cdc codec.Codec) error {
-	importedCrossChainGenState := crosschaintypes.GetGenesisStateFromAppState(cdc, importAppState)
-	importedCrossChainGenState.CrossChainTxs = importedCrossChainGenState.CrossChainTxs[:math.Min(MaxItemsForList, len(importedCrossChainGenState.CrossChainTxs))]
-	importedCrossChainGenState.InTxHashToCctxList = importedCrossChainGenState.InTxHashToCctxList[:math.Min(MaxItemsForList, len(importedCrossChainGenState.InTxHashToCctxList))]
-	importedCrossChainGenState.FinalizedInbounds = importedCrossChainGenState.FinalizedInbounds[:math.Min(MaxItemsForList, len(importedCrossChainGenState.FinalizedInbounds))]
-	importedCrossChainStateBz, err := json.Marshal(importedCrossChainGenState)
+	importedCrossChainGenState := crosschaintypes.GetGenesisStateFromAppStateLegacy(cdc, importAppState)
+	appStateGenState := crosschaintypes.GetGenesisStateFromAppState(cdc, appState)
+	// The genesis state has been modified between the two versions, so we add only the required fields and leave out the rest
+	// v16 adds the rate_limiter_flags and removes params from the genesis state
+	appStateGenState.CrossChainTxs = importedCrossChainGenState.CrossChainTxs[:math.Min(MaxItemsForList, len(importedCrossChainGenState.CrossChainTxs))]
+	appStateGenState.InTxHashToCctxList = importedCrossChainGenState.InTxHashToCctxList[:math.Min(MaxItemsForList, len(importedCrossChainGenState.InTxHashToCctxList))]
+	appStateGenState.FinalizedInbounds = importedCrossChainGenState.FinalizedInbounds[:math.Min(MaxItemsForList, len(importedCrossChainGenState.FinalizedInbounds))]
+	appStateBz, err := cdc.MarshalJSON(&appStateGenState)
 	if err != nil {
-		return fmt.Errorf("failed to marshal zetacrosschain genesis state: %w", err)
+		return fmt.Errorf("failed to marshal crosschain genesis state: %w", err)
 	}
-	appState[crosschaintypes.ModuleName] = importedCrossChainStateBz
+	appState[crosschaintypes.ModuleName] = appStateBz
 	return nil
 }
 

cmd/zetacored/parse_genesis_test.go

@@ -153,10 +153,10 @@ func ImportGenDoc(t *testing.T, cdc *codec.ProtoCodec, n int) *types.GenesisDoc
 }
 
 func GetImportData(t *testing.T, cdc *codec.ProtoCodec, n int) map[string]json.RawMessage {
-	importData := sample.AppState(t)
+	importData := sample.AppStateLegacy(t)
 
 	// Add crosschain data to genesis state
-	importedCrossChainGenState := crosschaintypes.GetGenesisStateFromAppState(cdc, importData)
+	importedCrossChainGenState := crosschaintypes.GetGenesisStateFromAppStateLegacy(cdc, importData)
 	cctxList := make([]*crosschaintypes.CrossChainTx, n)
 	intxHashToCctxList := make([]crosschaintypes.InTxHashToCctx, n)
 	finalLizedInbounds := make([]string, n)

contrib/localnet/docker-compose-import-data.yml

@@ -0,0 +1,145 @@
+version: "3"
+
+# This docker-compose file configures the localnet environment
+# it contains the following services:
+# - ZetaChain with 2 nodes (zetacore0, zetacore1)
+# - A observer set with 2 clients (zetaclient0, zetaclient1)
+# - An Ethereum node (eth)
+# - A Bitcoin node (bitcoin)
+# - A Rosetta API (rosetta)
+# - An orchestrator to manage interaction with the localnet (orchestrator)
+
+networks:
+  mynetwork:
+    ipam:
+      config:
+        - subnet: 172.20.0.0/24
+
+services:
+  rosetta:
+    image: zetanode:latest
+    container_name: rosetta
+    hostname: rosetta
+    ports:
+      - "8080:8080"
+    networks:
+      mynetwork:
+        ipv4_address: 172.20.0.200
+    entrypoint: ["zetacored", "rosetta", "--tendermint", "zetacore0:26657", "--grpc", "zetacore0:9090", "--network", "athens_101-1", "--blockchain",  "zetacore" ,"--retries", "500"]
+
+  zetacore0:
+    image: zetanode:latest
+    container_name: zetacore0
+    build:
+      context: ..
+      dockerfile: Dockerfile-localnet
+    hostname: zetacore0
+    ports:
+      - "1317:1317"
+      - "9545:8545"
+      - "9546:8546"
+      - "26657:26657"
+      - "6060:6060"
+      - "9090:9090"
+    networks:
+      mynetwork:
+        ipv4_address: 172.20.0.11
+    entrypoint: ["/root/start-zetacored.sh", "2","import-data"]
+    environment:
+      - HOTKEY_BACKEND=file
+      - HOTKEY_PASSWORD=password # test purposes only
+    volumes:
+      - ~/genesis_export/:/root/genesis_data
+
+  zetacore1:
+    image: zetanode:latest
+    container_name: zetacore1
+    build:
+      context: ..
+      dockerfile: Dockerfile-localnet
+    hostname: zetacore1
+    networks:
+      mynetwork:
+        ipv4_address: 172.20.0.12
+    entrypoint: ["/root/start-zetacored.sh", "2","import-data"]
+    environment:
+      - HOTKEY_BACKEND=file
+      - HOTKEY_PASSWORD=password # test purposes only
+
+
+  zetaclient0:
+    image: zetanode:latest
+    container_name: zetaclient0
+    build:
+      context: ..
+      dockerfile: Dockerfile-localnet
+    hostname: zetaclient0
+    networks:
+      mynetwork:
+        ipv4_address: 172.20.0.21
+    entrypoint: ["/root/start-zetaclientd.sh", "import-data"]
+    environment:
+      - ETHDEV_ENDPOINT=http://eth:8545
+      - HOTKEY_BACKEND=file
+      - HOTKEY_PASSWORD=password # test purposes only
+
+  zetaclient1:
+    image: zetanode:latest
+    container_name: zetaclient1
+    build:
+      context: ..
+      dockerfile: Dockerfile-localnet
+    hostname: zetaclient1
+    networks:
+      mynetwork:
+        ipv4_address: 172.20.0.22
+    entrypoint: ["/root/start-zetaclientd.sh", "import-data"]
+    environment:
+      - ETHDEV_ENDPOINT=http://eth:8545
+      - HOTKEY_BACKEND=file
+      - HOTKEY_PASSWORD=password # test purposes only
+
+  eth:
+    image: ethereum/client-go:v1.10.26
+    container_name: eth
+    hostname: eth
+    ports:
+      - "8545:8545"
+    networks:
+      mynetwork:
+        ipv4_address: 172.20.0.100
+    entrypoint: ["geth", "--dev", "--http", "--http.addr", "172.20.0.100", "--http.vhosts", "*", "--http.api", "eth,web3,net", "--http.corsdomain", "https://remix.ethereum.org", "--dev.period", "2"]
+
+  bitcoin:
+    image: ruimarinho/bitcoin-core:22 # version 23 is not working with btcd 0.22.0 due to change in createwallet rpc
+    container_name: bitcoin
+    hostname: bitcoin
+    networks:
+      mynetwork:
+        ipv4_address: 172.20.0.101
+    ports:
+      - "18443:18443"
+    command:
+      -printtoconsole
+      -regtest=1
+      -rpcallowip=0.0.0.0/0
+      -rpcbind=0.0.0.0
+      -rpcauth=smoketest:63acf9b8dccecce914d85ff8c044b78b$$5892f9bbc84f4364e79f0970039f88bdd823f168d4acc76099ab97b14a766a99
+      -txindex=1
+
+  orchestrator:
+    image: orchestrator:latest
+    tty: true
+    container_name: orchestrator
+    build:
+      context: ..
+      dockerfile: contrib/localnet/orchestrator/Dockerfile
+    depends_on:
+      - zetacore0
+      - eth
+    hostname: orchestrator
+    networks:
+      mynetwork:
+        ipv4_address: 172.20.0.2
+    entrypoint: ["/work/start-zetae2e.sh", "local","import-data"]
+

contrib/localnet/docker-compose.yml

@@ -64,6 +64,7 @@ services:
       - HOTKEY_BACKEND=file
       - HOTKEY_PASSWORD=password # test purposes only
 
+
   zetaclient0:
     image: zetanode:latest
     container_name: zetaclient0

contrib/localnet/orchestrator/start-zetae2e.sh

@@ -9,7 +9,13 @@ ZETAE2E_CMD=$1
 OPTION=$2
 
 echo "waiting for geth RPC to start..."
-sleep 2
+#  pause nodes other than zetacore0 to wait for zetacore0 to create genesis.json
+#  additional pause time is needed for importing data into the genesis as the export file is read into memory
+ if [ "$OPTION" != "import-data" ]; then
+    sleep 2
+  else
+    sleep 520
+  fi
 
 ### Create the accounts and fund them with Ether on local Ethereum network
 

contrib/localnet/scripts/import-data.sh

@@ -0,0 +1,15 @@
+#!/bin/bash
+if [ $# -lt 1 ]
+then
+  echo "Usage: import-data.sh [network]"
+  exit 1
+fi
+
+NETWORK=$1
+echo "NETWORK: ${NETWORK}"
+rm -rf ~/genesis_export/
+mkdir ~/genesis_export/
+echo "Download Latest State Export"
+LATEST_EXPORT_URL=$(curl https://snapshots.zetachain.com/latest-state-export | jq -r ."${NETWORK}")
+echo "LATEST EXPORT URL: ${LATEST_EXPORT_URL}"
+wget -q ${LATEST_EXPORT_URL} -O ~/genesis_export/exported-genesis.json

contrib/localnet/scripts/start-zetaclientd.sh

@@ -25,7 +25,14 @@ num=$(echo $HOSTNAME | tr -dc '0-9')
 node="zetacore$num"
 
 echo "Wait for zetacore to exchange genesis file"
-sleep 40
+#  pause nodes other than zetacore0 to wait for zetacore0 to create genesis.json
+#  additional pause time is needed for importing data into the genesis as the export file is read into memory
+if [ "$OPTION" != "import-data" ]; then
+  sleep 40
+else
+  sleep 510
+fi
+
 operator=$(cat $HOME/.zetacored/os.json | jq '.ObserverAddress' )
 operatorAddress=$(echo "$operator" | tr -d '"')
 echo "operatorAddress: $operatorAddress"

contrib/localnet/scripts/start-zetacored.sh

@@ -69,8 +69,15 @@ source ~/add-keys.sh
 # Pause other nodes so that the primary can node can do the genesis creation
 if [ $HOSTNAME != "zetacore0" ]
 then
+#  pause nodes other than zetacore0 to wait for zetacore0 to create genesis.json
+#  additional pause time is needed for importing data into the genesis as the export file is read into memory
+
   echo "Waiting for zetacore0 to create genesis.json"
-  sleep 10
+  if [ "$OPTION" != "import-data" ]; then
+    sleep 10
+  else
+    sleep 500
+  fi
   echo "genesis.json created"
 fi
 
@@ -155,9 +162,17 @@ then
 
 # 4. Collect all the gentx files in zetacore0 and create the final genesis.json
   zetacored collect-gentxs
+
+  if [ "$OPTION" == "import-data" ]; then
+    echo "Importing data"
+    zetacored parse-genesis-file /root/genesis_data/exported-genesis.json
+  fi
+
   zetacored validate-genesis
+
 # 5. Copy the final genesis.json to all the nodes
   for NODE in "${NODELIST[@]}"; do
+      echo "Copying genesis.json to $NODE"
       ssh $NODE rm -rf ~/.zetacored/genesis.json
       scp ~/.zetacored/config/genesis.json $NODE:~/.zetacored/config/genesis.json
   done