feat: add persistent localnet docker-compose

orchestrator persistence

geth and orchestrator persistence

also persist zetaclient dir (observer databases)

add extra-evm-chains.json merge into zetaclient config

.gitignore

@@ -55,4 +55,5 @@ contrib/localnet/grafana/addresses.txt
 contrib/localnet/addresses.txt
 
 # Config for e2e tests
-e2e_conf*
+e2e_conf*
+contrib/localnet/scripts/extra-evm-chains.json

contrib/localnet/docker-compose-persistent.yml

@@ -0,0 +1,71 @@
+# This docker-compose updates the services to use a persistent data directory
+# clear these volumes with this command: docker volume rm $(docker volume ls -q | grep -- '-persist$')
+
+services:
+  zetacore0:
+    volumes:
+      - zetacore0-zetacored-persist:/root/.zetacored
+
+  zetacore1:
+    volumes:
+      - zetacore1-zetacored-persist:/root/.zetacored
+
+  zetacore2:
+    volumes:
+      - zetacore2-zetacored-persist:/root/.zetacored
+
+  zetacore3:
+    volumes:
+      - zetacore3-zetacored-persist:/root/.zetacored
+
+  zetaclient0:
+    volumes:
+      - zetaclient0-zetacored-persist:/root/.zetacored
+      - zetaclient0-tss-persist:/root/.tss
+      - zetaclient0-zetaclient-persist:/root/.zetaclient
+
+  zetaclient1:
+    volumes:
+      - zetaclient1-zetacored-persist:/root/.zetacored
+      - zetaclient1-tss-persist:/root/.tss
+      - zetaclient1-zetaclient-persist:/root/.zetaclient
+
+  zetaclient2:
+    volumes:
+      - zetaclient2-zetacored-persist:/root/.zetacored
+      - zetaclient2-tss-persist:/root/.tss
+      - zetaclient2-zetaclient-persist:/root/.zetaclient
+
+  zetaclient3:
+    volumes:
+      - zetaclient3-zetacored-persist:/root/.zetacored
+      - zetaclient3-tss-persist:/root/.tss
+      - zetaclient3-zetaclient-persist:/root/.zetaclient
+
+  eth:
+    volumes:
+      - eth-data-persist:/root/data
+
+  orchestrator:
+    volumes:
+      - orchestrator-state-persist:/root/state
+
+volumes:
+  zetacore0-zetacored-persist:
+  zetacore1-zetacored-persist:
+  zetacore2-zetacored-persist:
+  zetacore3-zetacored-persist:
+  zetaclient0-zetacored-persist:
+  zetaclient0-tss-persist:
+  zetaclient0-zetaclient-persist:
+  zetaclient1-zetacored-persist:
+  zetaclient1-tss-persist:
+  zetaclient1-zetaclient-persist:
+  zetaclient2-zetacored-persist:
+  zetaclient2-tss-persist:
+  zetaclient2-zetaclient-persist:
+  zetaclient3-zetacored-persist:
+  zetaclient3-tss-persist:
+  zetaclient3-zetaclient-persist:
+  eth-data-persist:
+  orchestrator-state-persist:

contrib/localnet/docker-compose.yml

@@ -193,7 +193,7 @@ services:
     networks:
       mynetwork:
         ipv4_address: 172.20.0.100
-    entrypoint: ["geth", "--dev", "--http", "--http.addr", "172.20.0.100", "--http.vhosts", "*", "--http.api", "eth,web3,net", "--http.corsdomain", "https://remix.ethereum.org", "--dev.period", "2"]
+    entrypoint: ["geth", "--dev", "--datadir", "/root/data", "--http", "--http.addr", "172.20.0.100", "--http.vhosts", "*", "--http.api", "eth,web3,net", "--http.corsdomain", "https://remix.ethereum.org", "--dev.period", "2", "--"]
 
   eth2:
     build:

contrib/localnet/orchestrator/start-zetae2e.sh

@@ -160,14 +160,18 @@ if host ton > /dev/null; then
   ./wait-for-ton.sh
 fi
 
+# need to make the directory if it was not mounted as a volume
+mkdir -p /root/state
+deployed_config_path=/root/state/deployed.yml
+
 ### Run zetae2e command depending on the option passed
 
 # Mode migrate is used to run the e2e tests before and after the TSS migration
 # It runs the e2e tests with the migrate flag which triggers a TSS migration at the end of the tests. Once the migrationis done the first e2e test is complete
 # The second e2e test is run after the migration to ensure the network is still working as expected with the new tss address
 if [ "$LOCALNET_MODE" == "tss-migrate" ]; then
-  if [[ ! -f deployed.yml ]]; then
-    zetae2e local $E2E_ARGS --setup-only --config config.yml --config-out deployed.yml --skip-header-proof
+  if [[ ! -f "$deployed_config_path" ]]; then
+    zetae2e local $E2E_ARGS --setup-only --config config.yml --config-out "$deployed_config_path" --skip-header-proof
     if [ $? -ne 0 ]; then
       echo "e2e setup failed"
       exit 1
@@ -177,7 +181,7 @@ if [ "$LOCALNET_MODE" == "tss-migrate" ]; then
   fi
 
   echo "running e2e test before migrating TSS"
-  zetae2e local $E2E_ARGS --skip-setup --config deployed.yml --skip-header-proof
+  zetae2e local $E2E_ARGS --skip-setup --config "$deployed_config_path"  --skip-header-proof
   if [ $? -ne 0 ]; then
     echo "first e2e failed"
     exit 1
@@ -186,7 +190,7 @@ if [ "$LOCALNET_MODE" == "tss-migrate" ]; then
   echo "waiting 10 seconds for node to restart"
     sleep 10
 
-  zetae2e local --skip-setup --config deployed.yml --skip-bitcoin-setup --light --skip-header-proof
+  zetae2e local --skip-setup --config "$deployed_config_path" --skip-bitcoin-setup --light --skip-header-proof
   ZETAE2E_EXIT_CODE=$?
   if [ $ZETAE2E_EXIT_CODE -eq 0 ]; then
     echo "E2E passed after migration"
@@ -210,8 +214,8 @@ if [ "$LOCALNET_MODE" == "upgrade" ]; then
   OLD_VERSION=$(get_zetacored_version)
   COMMON_ARGS="--skip-header-proof --skip-tracker-check"
 
-  if [[ ! -f deployed.yml ]]; then
-    zetae2e local $E2E_ARGS --setup-only --config config.yml --config-out deployed.yml ${COMMON_ARGS}
+  if [[ ! -f "$deployed_config_path"  ]]; then
+    zetae2e local $E2E_ARGS --setup-only --config config.yml --config-out "$deployed_config_path"  ${COMMON_ARGS}
     if [ $? -ne 0 ]; then
       echo "e2e setup failed"
       exit 1
@@ -225,7 +229,7 @@ if [ "$LOCALNET_MODE" == "upgrade" ]; then
     echo "running E2E command to setup the networks and populate the state..."
 
     # Use light flag to ensure tests can complete before the upgrade height
-    zetae2e local $E2E_ARGS --skip-setup --config deployed.yml --light --skip-precompiles ${COMMON_ARGS}
+    zetae2e local $E2E_ARGS --skip-setup --config "$deployed_config_path"  --light --skip-precompiles ${COMMON_ARGS}
     if [ $? -ne 0 ]; then
       echo "first e2e failed"
       exit 1
@@ -264,9 +268,9 @@ if [ "$LOCALNET_MODE" == "upgrade" ]; then
   # When the upgrade height is greater than 100 for upgrade test, the Bitcoin tests have been run once, therefore the Bitcoin wallet is already set up
   # Use light flag to skip advanced tests
   if [ "$UPGRADE_HEIGHT" -lt 100 ]; then
-    zetae2e local $E2E_ARGS --skip-setup --config deployed.yml --light ${COMMON_ARGS}
+    zetae2e local $E2E_ARGS --skip-setup --config "$deployed_config_path" --light ${COMMON_ARGS}
   else
-    zetae2e local $E2E_ARGS --skip-setup --config deployed.yml --skip-bitcoin-setup --light ${COMMON_ARGS}
+    zetae2e local $E2E_ARGS --skip-setup --config "$deployed_config_path" --skip-bitcoin-setup --light ${COMMON_ARGS}
   fi
 
   ZETAE2E_EXIT_CODE=$?
@@ -282,8 +286,8 @@ else
   # If no mode is passed, run the e2e tests normally
   echo "running e2e setup..."
 
-  if [[ ! -f deployed.yml ]]; then
-    zetae2e local $E2E_ARGS --config config.yml --setup-only --config-out deployed.yml
+  if [[ ! -f "$deployed_config_path"  ]]; then
+    zetae2e local $E2E_ARGS --config config.yml --setup-only --config-out "$deployed_config_path"
     if [ $? -ne 0 ]; then
       echo "e2e setup failed"
       exit 1
@@ -298,7 +302,7 @@ else
 
   echo "running e2e tests with arguments: $E2E_ARGS"
 
-  zetae2e local $E2E_ARGS --skip-setup --config deployed.yml
+  zetae2e local $E2E_ARGS --skip-setup --config "$deployed_config_path"
   ZETAE2E_EXIT_CODE=$?
 
   # if e2e passed, exit with 0, otherwise exit with 1

contrib/localnet/scripts/start-zetaclientd.sh

@@ -109,4 +109,10 @@ then
   fi
 fi
 
+# merge extra-evm-chains.json into zetaclient_config.json if specified
+if [[ -f /root/extra-evm-chains.json ]]; then
+  jq '.EVMChainConfigs *= input' /root/.zetacored/config/zetaclient_config.json /root/extra-evm-chains.json > /tmp/merged_config.json
+  mv /tmp/merged_config.json /root/.zetacored/config/zetaclient_config.json
+fi
+
 zetaclientd-supervisor start < /root/password.file