diff --git a/Makefile b/Makefile index a97bde5c3a..0520f1f319 100644 --- a/Makefile +++ b/Makefile @@ -255,6 +255,11 @@ start-localnet-skip-build: cd contrib/localnet/ && $(DOCKER) compose -f docker-compose.yml -f docker-compose-setup-only.yml up -d stop-localnet: +start-e2e-import-mainnet-test: zetanode + @echo "--> Starting e2e import-data test" + cd contrib/localnet/ && ./scripts/import-data.sh mainnet && $(DOCKER) compose -f docker-compose.yml -f docker-compose-import-data.yml up -d + +stop-test: cd contrib/localnet/ && $(DOCKER) compose down --remove-orphans ############################################################################### diff --git a/changelog.md b/changelog.md index 37224c817d..42de5f2e74 100644 --- a/changelog.md +++ b/changelog.md @@ -63,6 +63,7 @@ * [2329](https://github.com/zeta-chain/node/pull/2329) - fix TODOs in rpc unit tests * [2342](https://github.com/zeta-chain/node/pull/2342) - extend rpc unit tests with testing extension to include synthetic ethereum txs * [2299](https://github.com/zeta-chain/node/pull/2299) - add `zetae2e` command to deploy test contracts +* [2360](https://github.com/zeta-chain/node/pull/2360) - add stateful e2e tests. * [2349](https://github.com/zeta-chain/node/pull/2349) - add TestBitcoinDepositRefund and WithdrawBitcoinMultipleTimes E2E tests ### Fixes diff --git a/cmd/zetacored/parse_genesis.go b/cmd/zetacored/parse_genesis.go index b2a4258995..ec0a9bd313 100644 --- a/cmd/zetacored/parse_genesis.go +++ b/cmd/zetacored/parse_genesis.go @@ -41,7 +41,6 @@ const MaxItemsForList = 10 // Copy represents a set of modules for which, the entire state is copied without any modifications var Copy = map[string]bool{ slashingtypes.ModuleName: true, - govtypes.ModuleName: true, crisistypes.ModuleName: true, feemarkettypes.ModuleName: true, paramstypes.ModuleName: true, @@ -50,24 +49,37 @@ var Copy = map[string]bool{ vestingtypes.ModuleName: true, fungibletypes.ModuleName: true, emissionstypes.ModuleName: true, - authz.ModuleName: true, } // Skip represents a set of modules for which, the entire state is skipped and nothing gets imported var Skip = map[string]bool{ - evmtypes.ModuleName: true, - stakingtypes.ModuleName: true, - genutiltypes.ModuleName: true, - authtypes.ModuleName: true, - banktypes.ModuleName: true, + // Skipping evm this is done to reduce the size of the genesis file evm module uses the majority of the space due to smart contract data + evmtypes.ModuleName: true, + // Skipping staking as new validators would be created for the new chain + stakingtypes.ModuleName: true, + // Skipping genutil as new gentxs would be created + genutiltypes.ModuleName: true, + // Skipping auth as new accounts would be created for the new chain. This also needs to be done as we are skipping evm module + authtypes.ModuleName: true, + // Skipping bank module as it is not used when starting a new chain this is done to make sure the total supply invariant is maintained. + // This would need modification but might be possible to add in non evm based modules in the future + banktypes.ModuleName: true, + // Skipping distribution module as it is not used when starting a new chain , rewards are based on validators and delegators , and so rewards from a different chain do not hold any value distributiontypes.ModuleName: true, - group.ModuleName: true, + // Skipping group module as it is not used when starting a new chain, new groups should be created based on the validator operator keys + group.ModuleName: true, + // Skipping authz as it is not used when starting a new chain, new grants should be created based on the validator hotkeys abd operator keys + authz.ModuleName: true, + // Skipping fungible module as new fungible tokens would be created and system contract would be deployed + fungibletypes.ModuleName: true, + // Skipping gov types as new parameters are set for the new chain + govtypes.ModuleName: true, } // Modify represents a set of modules for which, the state is modified before importing. Each Module should have a corresponding Modify function var Modify = map[string]bool{ - crosschaintypes.ModuleName: true, observertypes.ModuleName: true, + crosschaintypes.ModuleName: true, } func CmdParseGenesisFile() *cobra.Command { @@ -78,6 +90,10 @@ func CmdParseGenesisFile() *cobra.Command { RunE: func(cmd *cobra.Command, args []string) error { clientCtx := client.GetClientContextFromCmd(cmd) cdc := clientCtx.Codec + modifyEnabled, err := cmd.Flags().GetBool("modify") + if err != nil { + return err + } genesisFilePath := filepath.Join(app.DefaultNodeHome, "config", "genesis.json") if len(args) == 2 { genesisFilePath = args[1] @@ -90,7 +106,7 @@ func CmdParseGenesisFile() *cobra.Command { if err != nil { return err } - err = ImportDataIntoFile(genDoc, importData, cdc) + err = ImportDataIntoFile(genDoc, importData, cdc, modifyEnabled) if err != nil { return err } @@ -103,10 +119,16 @@ func CmdParseGenesisFile() *cobra.Command { return nil }, } + cmd.PersistentFlags().Bool("modify", false, "Modify the genesis file before importing") return cmd } -func ImportDataIntoFile(genDoc *types.GenesisDoc, importFile *types.GenesisDoc, cdc codec.Codec) error { +func ImportDataIntoFile( + genDoc *types.GenesisDoc, + importFile *types.GenesisDoc, + cdc codec.Codec, + modifyEnabled bool, +) error { appState, err := genutiltypes.GenesisStateFromGenDoc(*genDoc) if err != nil { @@ -124,7 +146,7 @@ func ImportDataIntoFile(genDoc *types.GenesisDoc, importFile *types.GenesisDoc, if Copy[m] { appState[m] = importAppState[m] } - if Modify[m] { + if Modify[m] && modifyEnabled { switch m { case crosschaintypes.ModuleName: err := ModifyCrosschainState(appState, importAppState, cdc) diff --git a/cmd/zetacored/parse_genesis_test.go b/cmd/zetacored/parse_genesis_test.go index 7eb75f4a0f..689075eb32 100644 --- a/cmd/zetacored/parse_genesis_test.go +++ b/cmd/zetacored/parse_genesis_test.go @@ -92,43 +92,86 @@ func Test_ModifyObserverState(t *testing.T) { } func Test_ImportDataIntoFile(t *testing.T) { - setConfig(t) - cdc := keepertest.NewCodec() - genDoc := sample.GenDoc(t) - importGenDoc := ImportGenDoc(t, cdc, 100) + t.Run("successfully import data into file and modify data", func(t *testing.T) { + setConfig(t) + cdc := keepertest.NewCodec() + genDoc := sample.GenDoc(t) + importGenDoc := ImportGenDoc(t, cdc, 100) - err := zetacored.ImportDataIntoFile(genDoc, importGenDoc, cdc) - require.NoError(t, err) + err := zetacored.ImportDataIntoFile(genDoc, importGenDoc, cdc, true) + require.NoError(t, err) - appState, err := genutiltypes.GenesisStateFromGenDoc(*genDoc) - require.NoError(t, err) + appState, err := genutiltypes.GenesisStateFromGenDoc(*genDoc) + require.NoError(t, err) - // Crosschain module is in Modify list - crosschainStateAfterImport := crosschaintypes.GetGenesisStateFromAppState(cdc, appState) - require.Len(t, crosschainStateAfterImport.CrossChainTxs, zetacored.MaxItemsForList) - require.Len(t, crosschainStateAfterImport.InboundHashToCctxList, zetacored.MaxItemsForList) - require.Len(t, crosschainStateAfterImport.FinalizedInbounds, zetacored.MaxItemsForList) + // Crosschain module is in Modify list + crosschainStateAfterImport := crosschaintypes.GetGenesisStateFromAppState(cdc, appState) + require.Len(t, crosschainStateAfterImport.CrossChainTxs, zetacored.MaxItemsForList) + require.Len(t, crosschainStateAfterImport.InboundHashToCctxList, zetacored.MaxItemsForList) + require.Len(t, crosschainStateAfterImport.FinalizedInbounds, zetacored.MaxItemsForList) - // Bank module is in Skip list - var bankStateAfterImport banktypes.GenesisState - if appState[banktypes.ModuleName] != nil { - err := cdc.UnmarshalJSON(appState[banktypes.ModuleName], &bankStateAfterImport) - if err != nil { - panic(fmt.Sprintf("Failed to get genesis state from app state: %s", err.Error())) + // Bank module is in Skip list + var bankStateAfterImport banktypes.GenesisState + if appState[banktypes.ModuleName] != nil { + err := cdc.UnmarshalJSON(appState[banktypes.ModuleName], &bankStateAfterImport) + if err != nil { + panic(fmt.Sprintf("Failed to get genesis state from app state: %s", err.Error())) + } } - } - // 4 balances were present in the original genesis state - require.Len(t, bankStateAfterImport.Balances, 4) + // 4 balances were present in the original genesis state + require.Len(t, bankStateAfterImport.Balances, 4) - // Emissions module is in Copy list - var emissionStateAfterImport emissionstypes.GenesisState - if appState[emissionstypes.ModuleName] != nil { - err := cdc.UnmarshalJSON(appState[emissionstypes.ModuleName], &emissionStateAfterImport) - if err != nil { - panic(fmt.Sprintf("Failed to get genesis state from app state: %s", err.Error())) + // Emissions module is in Copy list + var emissionStateAfterImport emissionstypes.GenesisState + if appState[emissionstypes.ModuleName] != nil { + err := cdc.UnmarshalJSON(appState[emissionstypes.ModuleName], &emissionStateAfterImport) + if err != nil { + panic(fmt.Sprintf("Failed to get genesis state from app state: %s", err.Error())) + } } - } - require.Len(t, emissionStateAfterImport.WithdrawableEmissions, 100) + require.Len(t, emissionStateAfterImport.WithdrawableEmissions, 100) + }) + + t.Run("successfully import data into file without modifying data", func(t *testing.T) { + setConfig(t) + cdc := keepertest.NewCodec() + genDoc := sample.GenDoc(t) + importGenDoc := ImportGenDoc(t, cdc, 8) + + err := zetacored.ImportDataIntoFile(genDoc, importGenDoc, cdc, false) + require.NoError(t, err) + + appState, err := genutiltypes.GenesisStateFromGenDoc(*genDoc) + require.NoError(t, err) + + // Crosschain module is in Modify list + crosschainStateAfterImport := crosschaintypes.GetGenesisStateFromAppState(cdc, appState) + require.Len(t, crosschainStateAfterImport.CrossChainTxs, 0) + require.Len(t, crosschainStateAfterImport.InboundHashToCctxList, 0) + require.Len(t, crosschainStateAfterImport.FinalizedInbounds, 0) + + // Bank module is in Skip list + var bankStateAfterImport banktypes.GenesisState + if appState[banktypes.ModuleName] != nil { + err := cdc.UnmarshalJSON(appState[banktypes.ModuleName], &bankStateAfterImport) + if err != nil { + panic(fmt.Sprintf("Failed to get genesis state from app state: %s", err.Error())) + } + } + // 4 balances were present in the original genesis state + require.Len(t, bankStateAfterImport.Balances, 4) + + // Emissions module is in Copy list + var emissionStateAfterImport emissionstypes.GenesisState + if appState[emissionstypes.ModuleName] != nil { + err := cdc.UnmarshalJSON(appState[emissionstypes.ModuleName], &emissionStateAfterImport) + if err != nil { + panic(fmt.Sprintf("Failed to get genesis state from app state: %s", err.Error())) + } + } + require.Len(t, emissionStateAfterImport.WithdrawableEmissions, 8) + + }) } func Test_GetGenDoc(t *testing.T) { diff --git a/contrib/localnet/docker-compose-import-data.yml b/contrib/localnet/docker-compose-import-data.yml new file mode 100644 index 0000000000..19f4068ad5 --- /dev/null +++ b/contrib/localnet/docker-compose-import-data.yml @@ -0,0 +1,30 @@ +version: "3" + +# This docker-compose file configures the localnet environment +# it contains the following services: +# - ZetaChain with 2 nodes (zetacore0, zetacore1) +# - A observer set with 2 clients (zetaclient0, zetaclient1) +# - An Ethereum node (eth) +# - A Bitcoin node (bitcoin) +# - A Rosetta API (rosetta) +# - An orchestrator to manage interaction with the localnet (orchestrator) +services: + rosetta: + entrypoint: ["/root/start-rosetta.sh"] + + zetacore0: + entrypoint: ["/root/start-zetacored.sh", "2","import-data"] + volumes: + - ~/genesis_export/:/root/genesis_data + + zetacore1: + entrypoint: ["/root/start-zetacored.sh", "2","import-data"] + + zetaclient0: + entrypoint: ["/root/start-zetaclientd.sh"] + + zetaclient1: + entrypoint: ["/root/start-zetaclientd.sh"] + + orchestrator: + entrypoint: ["/work/start-zetae2e.sh", "local"] \ No newline at end of file diff --git a/contrib/localnet/orchestrator/start-zetae2e.sh b/contrib/localnet/orchestrator/start-zetae2e.sh index a297be595b..8dcb4ea271 100644 --- a/contrib/localnet/orchestrator/start-zetae2e.sh +++ b/contrib/localnet/orchestrator/start-zetae2e.sh @@ -29,6 +29,12 @@ while [ ! -f ~/.ssh/authorized_keys ]; do sleep 1 done +# need to wait for zetacore0 to be up +while ! curl -s -o /dev/null zetacore0:26657/status ; do + echo "Waiting for zetacore0 rpc" + sleep 10 +done + echo "waiting for geth RPC to start..." sleep 2 diff --git a/contrib/localnet/scripts/import-data.sh b/contrib/localnet/scripts/import-data.sh new file mode 100644 index 0000000000..d71d5c3656 --- /dev/null +++ b/contrib/localnet/scripts/import-data.sh @@ -0,0 +1,15 @@ +#!/bin/bash +if [ $# -lt 1 ] +then + echo "Usage: import-data.sh [network]" + exit 1 +fi + +NETWORK=$1 +echo "NETWORK: ${NETWORK}" +rm -rf ~/genesis_export/ +mkdir ~/genesis_export/ +echo "Download Latest State Export" +LATEST_EXPORT_URL=$(curl https://snapshots.zetachain.com/latest-state-export | jq -r ."${NETWORK}") +echo "LATEST EXPORT URL: ${LATEST_EXPORT_URL}" +wget -q ${LATEST_EXPORT_URL} -O ~/genesis_export/exported-genesis.json \ No newline at end of file diff --git a/contrib/localnet/scripts/start-rosetta.sh b/contrib/localnet/scripts/start-rosetta.sh new file mode 100644 index 0000000000..e675da6a6a --- /dev/null +++ b/contrib/localnet/scripts/start-rosetta.sh @@ -0,0 +1,14 @@ +#!/bin/bash + +# This script is used to start the Rosetta API server for the Zetacore network. + +echo "Waiting for network to start producing blocks" +CURRENT_HEIGHT=0 +WAIT_HEIGHT=1 +while [[ $CURRENT_HEIGHT -lt $WAIT_HEIGHT ]] +do + CURRENT_HEIGHT=$(curl -s zetacore0:26657/status | jq '.result.sync_info.latest_block_height' | tr -d '"') + sleep 5 +done + +zetacored rosetta --tendermint zetacore0:26657 --grpc zetacore0:9090 --network athens_101-1 --blockchain zetacore \ No newline at end of file diff --git a/contrib/localnet/scripts/start-zetaclientd.sh b/contrib/localnet/scripts/start-zetaclientd.sh index 01667df450..1b018de4ef 100755 --- a/contrib/localnet/scripts/start-zetaclientd.sh +++ b/contrib/localnet/scripts/start-zetaclientd.sh @@ -28,6 +28,15 @@ while [ ! -f ~/.ssh/authorized_keys ]; do sleep 1 done + + +# need to wait for zetacore0 to be up +while ! curl -s -o /dev/null zetacore0:26657/status ; do + echo "Waiting for zetacore0 rpc" + sleep 10 +done + + # read HOTKEY_BACKEND env var for hotkey keyring backend and set default to test BACKEND="test" if [ "$HOTKEY_BACKEND" == "file" ]; then diff --git a/contrib/localnet/scripts/start-zetacored.sh b/contrib/localnet/scripts/start-zetacored.sh index 8e16ffbc4a..d390b99083 100755 --- a/contrib/localnet/scripts/start-zetacored.sh +++ b/contrib/localnet/scripts/start-zetacored.sh @@ -4,6 +4,8 @@ # It initializes the nodes and creates the genesis.json file # It also starts the nodes # The number of nodes is passed as an first argument to the script +# The second argument is optional and can have the following value: +# - import-data: import data into the genesis file /usr/sbin/sshd @@ -71,6 +73,7 @@ then exit 1 fi NUMOFNODES=$1 +OPTION=$2 # create keys CHAINID="athens_101-1" @@ -254,6 +257,11 @@ then scp $NODE:~/.zetacored/config/gentx/* ~/.zetacored/config/gentx/z2gentx/ done + if [[ "$OPTION" == "import-data" || "$OPTION" == "import-data-upgrade" ]]; then + echo "Importing data" + zetacored parse-genesis-file /root/genesis_data/exported-genesis.json + fi + # 4. Collect all the gentx files in zetacore0 and create the final genesis.json zetacored collect-gentxs zetacored validate-genesis diff --git a/docs/cli/zetacored/zetacored_parse-genesis-file.md b/docs/cli/zetacored/zetacored_parse-genesis-file.md index 44bcb41424..a7b66ebb73 100644 --- a/docs/cli/zetacored/zetacored_parse-genesis-file.md +++ b/docs/cli/zetacored/zetacored_parse-genesis-file.md @@ -9,7 +9,8 @@ zetacored parse-genesis-file [import-genesis-file] [optional-genesis-file] [flag ### Options ``` - -h, --help help for parse-genesis-file + -h, --help help for parse-genesis-file + --modify Modify the genesis file before importing ``` ### Options inherited from parent commands