feat: add optional persistence to localnet (#2868)

* feat: add persistent localnet docker-compose

orchestrator persistence

geth and orchestrator persistence

also persist zetaclient dir (observer databases)

add extra-evm-chains.json merge into zetaclient config

* makefile examples

* add local testing docs

.gitignore

@@ -55,4 +55,5 @@ contrib/localnet/grafana/addresses.txt
 contrib/localnet/addresses.txt
 
 # Config for e2e tests
-e2e_conf*
+e2e_conf*
+contrib/localnet/scripts/extra-evm-chains.json

Makefile

@@ -5,9 +5,10 @@ VERSION := $(shell ./version.sh)
 COMMIT := $(shell [ -z "${COMMIT_ID}" ] && git log -1 --format='%H' || echo ${COMMIT_ID} )
 BUILDTIME := $(shell date -u +"%Y%m%d.%H%M%S" )
 DOCKER ?= docker
-# allow setting of DOCKER_COMPOSE_ARGS to pass additional args to docker compose
-# useful for setting profiles
-DOCKER_COMPOSE ?= $(DOCKER) compose $(COMPOSE_ARGS)
+# allow setting of NODE_COMPOSE_ARGS to pass additional args to docker compose
+# useful for setting profiles and/ort optional overlays
+# example: NODE_COMPOSE_ARGS="--profile monitoring -f docker-compose-persistent.yml"
+DOCKER_COMPOSE ?= $(DOCKER) compose $(NODE_COMPOSE_ARGS)
 DOCKER_BUF := $(DOCKER) run --rm -v $(CURDIR):/workspace --workdir /workspace bufbuild/buf
 GOFLAGS := ""
 GOLANG_CROSS_VERSION ?= v1.22.4
@@ -224,7 +225,7 @@ start-localnet-skip-build:
 
 # stop-localnet should include all profiles so other containers are also removed
 stop-localnet:
-	cd contrib/localnet/ && $(DOCKER_COMPOSE) --profile all down --remove-orphans
+	cd contrib/localnet/ && $(DOCKER_COMPOSE) --profile all -f docker-compose.yml down --remove-orphans
 
 ###############################################################################
 ###                         E2E tests               						###

contrib/localnet/docker-compose-persistent.yml

@@ -0,0 +1,71 @@
+# This docker-compose updates the services to use a persistent data directory
+# clear these volumes with this command: docker volume rm $(docker volume ls -q | grep -- '-persist$')
+
+services:
+  zetacore0:
+    volumes:
+      - zetacore0-zetacored-persist:/root/.zetacored
+
+  zetacore1:
+    volumes:
+      - zetacore1-zetacored-persist:/root/.zetacored
+
+  zetacore2:
+    volumes:
+      - zetacore2-zetacored-persist:/root/.zetacored
+
+  zetacore3:
+    volumes:
+      - zetacore3-zetacored-persist:/root/.zetacored
+
+  zetaclient0:
+    volumes:
+      - zetaclient0-zetacored-persist:/root/.zetacored
+      - zetaclient0-tss-persist:/root/.tss
+      - zetaclient0-zetaclient-persist:/root/.zetaclient
+
+  zetaclient1:
+    volumes:
+      - zetaclient1-zetacored-persist:/root/.zetacored
+      - zetaclient1-tss-persist:/root/.tss
+      - zetaclient1-zetaclient-persist:/root/.zetaclient
+
+  zetaclient2:
+    volumes:
+      - zetaclient2-zetacored-persist:/root/.zetacored
+      - zetaclient2-tss-persist:/root/.tss
+      - zetaclient2-zetaclient-persist:/root/.zetaclient
+
+  zetaclient3:
+    volumes:
+      - zetaclient3-zetacored-persist:/root/.zetacored
+      - zetaclient3-tss-persist:/root/.tss
+      - zetaclient3-zetaclient-persist:/root/.zetaclient
+
+  eth:
+    volumes:
+      - eth-data-persist:/root/data
+
+  orchestrator:
+    volumes:
+      - orchestrator-state-persist:/root/state
+
+volumes:
+  zetacore0-zetacored-persist:
+  zetacore1-zetacored-persist:
+  zetacore2-zetacored-persist:
+  zetacore3-zetacored-persist:
+  zetaclient0-zetacored-persist:
+  zetaclient0-tss-persist:
+  zetaclient0-zetaclient-persist:
+  zetaclient1-zetacored-persist:
+  zetaclient1-tss-persist:
+  zetaclient1-zetaclient-persist:
+  zetaclient2-zetacored-persist:
+  zetaclient2-tss-persist:
+  zetaclient2-zetaclient-persist:
+  zetaclient3-zetacored-persist:
+  zetaclient3-tss-persist:
+  zetaclient3-zetaclient-persist:
+  eth-data-persist:
+  orchestrator-state-persist:

contrib/localnet/docker-compose.yml

@@ -193,7 +193,7 @@ services:
     networks:
       mynetwork:
         ipv4_address: 172.20.0.100
-    entrypoint: ["geth", "--dev", "--http", "--http.addr", "172.20.0.100", "--http.vhosts", "*", "--http.api", "eth,web3,net", "--http.corsdomain", "https://remix.ethereum.org", "--dev.period", "2"]
+    entrypoint: ["geth", "--dev", "--datadir", "/root/data", "--http", "--http.addr", "172.20.0.100", "--http.vhosts", "*", "--http.api", "eth,web3,net", "--http.corsdomain", "https://remix.ethereum.org", "--dev.period", "2", "--"]
 
   eth2:
     build:
@@ -313,7 +313,7 @@ services:
       - ssh:/root/.ssh
 
   grafana:
-    image: grafana/grafana:9.1.7
+    image: grafana/grafana:10.4.8
     container_name: grafana
     hostname: grafana
     profiles:
@@ -322,6 +322,7 @@ services:
     volumes:
       - ./grafana/datasource.yaml:/etc/grafana/provisioning/datasources/datasource.yaml
       - ./grafana/dashboards/:/etc/grafana/provisioning/dashboards
+      - ./grafana/grafana.ini:/etc/grafana/grafana.ini
       - grafana_storage:/var/lib/grafana
     ports:
       - "3000:3000"

contrib/localnet/grafana/grafana.ini

@@ -0,0 +1,5 @@
+[auth.anonymous]
+enabled = true
+
+org_id = 1
+org_role = Editor

contrib/localnet/orchestrator/start-zetae2e.sh

@@ -160,14 +160,18 @@ if host ton > /dev/null; then
   ./wait-for-ton.sh
 fi
 
+# need to make the directory if it was not mounted as a volume
+mkdir -p /root/state
+deployed_config_path=/root/state/deployed.yml
+
 ### Run zetae2e command depending on the option passed
 
 # Mode migrate is used to run the e2e tests before and after the TSS migration
 # It runs the e2e tests with the migrate flag which triggers a TSS migration at the end of the tests. Once the migrationis done the first e2e test is complete
 # The second e2e test is run after the migration to ensure the network is still working as expected with the new tss address
 if [ "$LOCALNET_MODE" == "tss-migrate" ]; then
-  if [[ ! -f deployed.yml ]]; then
-    zetae2e local $E2E_ARGS --setup-only --config config.yml --config-out deployed.yml --skip-header-proof
+  if [[ ! -f "$deployed_config_path" ]]; then
+    zetae2e local $E2E_ARGS --setup-only --config config.yml --config-out "$deployed_config_path" --skip-header-proof
     if [ $? -ne 0 ]; then
       echo "e2e setup failed"
       exit 1
@@ -177,7 +181,7 @@ if [ "$LOCALNET_MODE" == "tss-migrate" ]; then
   fi
 
   echo "running e2e test before migrating TSS"
-  zetae2e local $E2E_ARGS --skip-setup --config deployed.yml --skip-header-proof
+  zetae2e local $E2E_ARGS --skip-setup --config "$deployed_config_path"  --skip-header-proof
   if [ $? -ne 0 ]; then
     echo "first e2e failed"
     exit 1
@@ -186,7 +190,7 @@ if [ "$LOCALNET_MODE" == "tss-migrate" ]; then
   echo "waiting 10 seconds for node to restart"
     sleep 10
 
-  zetae2e local --skip-setup --config deployed.yml --skip-bitcoin-setup --light --skip-header-proof
+  zetae2e local --skip-setup --config "$deployed_config_path" --skip-bitcoin-setup --light --skip-header-proof
   ZETAE2E_EXIT_CODE=$?
   if [ $ZETAE2E_EXIT_CODE -eq 0 ]; then
     echo "E2E passed after migration"
@@ -210,8 +214,8 @@ if [ "$LOCALNET_MODE" == "upgrade" ]; then
   OLD_VERSION=$(get_zetacored_version)
   COMMON_ARGS="--skip-header-proof --skip-tracker-check"
 
-  if [[ ! -f deployed.yml ]]; then
-    zetae2e local $E2E_ARGS --setup-only --config config.yml --config-out deployed.yml ${COMMON_ARGS}
+  if [[ ! -f "$deployed_config_path"  ]]; then
+    zetae2e local $E2E_ARGS --setup-only --config config.yml --config-out "$deployed_config_path"  ${COMMON_ARGS}
     if [ $? -ne 0 ]; then
       echo "e2e setup failed"
       exit 1
@@ -225,7 +229,7 @@ if [ "$LOCALNET_MODE" == "upgrade" ]; then
     echo "running E2E command to setup the networks and populate the state..."
 
     # Use light flag to ensure tests can complete before the upgrade height
-    zetae2e local $E2E_ARGS --skip-setup --config deployed.yml --light --skip-precompiles ${COMMON_ARGS}
+    zetae2e local $E2E_ARGS --skip-setup --config "$deployed_config_path"  --light --skip-precompiles ${COMMON_ARGS}
     if [ $? -ne 0 ]; then
       echo "first e2e failed"
       exit 1
@@ -264,9 +268,9 @@ if [ "$LOCALNET_MODE" == "upgrade" ]; then
   # When the upgrade height is greater than 100 for upgrade test, the Bitcoin tests have been run once, therefore the Bitcoin wallet is already set up
   # Use light flag to skip advanced tests
   if [ "$UPGRADE_HEIGHT" -lt 100 ]; then
-    zetae2e local $E2E_ARGS --skip-setup --config deployed.yml --light ${COMMON_ARGS}
+    zetae2e local $E2E_ARGS --skip-setup --config "$deployed_config_path" --light ${COMMON_ARGS}
   else
-    zetae2e local $E2E_ARGS --skip-setup --config deployed.yml --skip-bitcoin-setup --light ${COMMON_ARGS}
+    zetae2e local $E2E_ARGS --skip-setup --config "$deployed_config_path" --skip-bitcoin-setup --light ${COMMON_ARGS}
   fi
 
   ZETAE2E_EXIT_CODE=$?
@@ -282,8 +286,8 @@ else
   # If no mode is passed, run the e2e tests normally
   echo "running e2e setup..."
 
-  if [[ ! -f deployed.yml ]]; then
-    zetae2e local $E2E_ARGS --config config.yml --setup-only --config-out deployed.yml
+  if [[ ! -f "$deployed_config_path"  ]]; then
+    zetae2e local $E2E_ARGS --config config.yml --setup-only --config-out "$deployed_config_path"
     if [ $? -ne 0 ]; then
       echo "e2e setup failed"
       exit 1
@@ -298,7 +302,7 @@ else
 
   echo "running e2e tests with arguments: $E2E_ARGS"
 
-  zetae2e local $E2E_ARGS --skip-setup --config deployed.yml
+  zetae2e local $E2E_ARGS --skip-setup --config "$deployed_config_path"
   ZETAE2E_EXIT_CODE=$?
 
   # if e2e passed, exit with 0, otherwise exit with 1

contrib/localnet/scripts/start-zetaclientd.sh

@@ -109,4 +109,10 @@ then
   fi
 fi
 
+# merge extra-evm-chains.json into zetaclient_config.json if specified
+if [[ -f /root/extra-evm-chains.json ]]; then
+  jq '.EVMChainConfigs *= input' /root/.zetacored/config/zetaclient_config.json /root/extra-evm-chains.json > /tmp/merged_config.json
+  mv /tmp/merged_config.json /root/.zetacored/config/zetaclient_config.json
+fi
+
 zetaclientd-supervisor start < /root/password.file

docs/development/LOCAL_TESTING.md

@@ -38,6 +38,15 @@ This uses `docker compose` to start the localnet and run standard e2e tests insi
 - `ZETACORED_IMPORT_GENESIS_DATA`: path to genesis data to import before starting zetacored
 - `ZETACORED_START_PERIOD`: duration to tolerate `zetacored` health check failures during startup
 
+More options directly to `docker compose` via the `NODE_COMPOSE_ARGS` variable. This allows setting additional profiles or configuring an overlay. Example:
+
+```
+example: NODE_COMPOSE_ARGS="--profile monitoring -f docker-compose-persistent.yml"`
+make start-e2e-test
+```
+
+This starts the e2e tests while enabling the monitoring stack and persistence (data is not deleted between test runs).
+
 #### Run admin functions e2e tests
 
 We define e2e tests allowing to test admin functionalities (emergency network pause for example).