From c7ac9b4c11c9b834300e3e926af1ccde6e3c2318 Mon Sep 17 00:00:00 2001 From: Alex Gartner Date: Fri, 20 Sep 2024 11:13:15 -0700 Subject: [PATCH] feat: add optional persistence to localnet (#2868) * feat: add persistent localnet docker-compose orchestrator persistence geth and orchestrator persistence also persist zetaclient dir (observer databases) add extra-evm-chains.json merge into zetaclient config * makefile examples * add local testing docs --- .gitignore | 3 +- Makefile | 9 +-- .../localnet/docker-compose-persistent.yml | 71 +++++++++++++++++++ contrib/localnet/docker-compose.yml | 5 +- contrib/localnet/grafana/grafana.ini | 5 ++ .../localnet/orchestrator/start-zetae2e.sh | 28 ++++---- contrib/localnet/scripts/start-zetaclientd.sh | 6 ++ docs/development/LOCAL_TESTING.md | 9 +++ 8 files changed, 117 insertions(+), 19 deletions(-) create mode 100644 contrib/localnet/docker-compose-persistent.yml create mode 100644 contrib/localnet/grafana/grafana.ini diff --git a/.gitignore b/.gitignore index 080a8030b2..e0f51f8d4a 100644 --- a/.gitignore +++ b/.gitignore @@ -55,4 +55,5 @@ contrib/localnet/grafana/addresses.txt contrib/localnet/addresses.txt # Config for e2e tests -e2e_conf* \ No newline at end of file +e2e_conf* +contrib/localnet/scripts/extra-evm-chains.json \ No newline at end of file diff --git a/Makefile b/Makefile index 7648d07df4..b3fbf4009a 100644 --- a/Makefile +++ b/Makefile @@ -5,9 +5,10 @@ VERSION := $(shell ./version.sh) COMMIT := $(shell [ -z "${COMMIT_ID}" ] && git log -1 --format='%H' || echo ${COMMIT_ID} ) BUILDTIME := $(shell date -u +"%Y%m%d.%H%M%S" ) DOCKER ?= docker -# allow setting of DOCKER_COMPOSE_ARGS to pass additional args to docker compose -# useful for setting profiles -DOCKER_COMPOSE ?= $(DOCKER) compose $(COMPOSE_ARGS) +# allow setting of NODE_COMPOSE_ARGS to pass additional args to docker compose +# useful for setting profiles and/ort optional overlays +# example: NODE_COMPOSE_ARGS="--profile monitoring -f docker-compose-persistent.yml" +DOCKER_COMPOSE ?= $(DOCKER) compose $(NODE_COMPOSE_ARGS) DOCKER_BUF := $(DOCKER) run --rm -v $(CURDIR):/workspace --workdir /workspace bufbuild/buf GOFLAGS := "" GOLANG_CROSS_VERSION ?= v1.22.4 @@ -224,7 +225,7 @@ start-localnet-skip-build: # stop-localnet should include all profiles so other containers are also removed stop-localnet: - cd contrib/localnet/ && $(DOCKER_COMPOSE) --profile all down --remove-orphans + cd contrib/localnet/ && $(DOCKER_COMPOSE) --profile all -f docker-compose.yml down --remove-orphans ############################################################################### ### E2E tests ### diff --git a/contrib/localnet/docker-compose-persistent.yml b/contrib/localnet/docker-compose-persistent.yml new file mode 100644 index 0000000000..4dc5374d10 --- /dev/null +++ b/contrib/localnet/docker-compose-persistent.yml @@ -0,0 +1,71 @@ +# This docker-compose updates the services to use a persistent data directory +# clear these volumes with this command: docker volume rm $(docker volume ls -q | grep -- '-persist$') + +services: + zetacore0: + volumes: + - zetacore0-zetacored-persist:/root/.zetacored + + zetacore1: + volumes: + - zetacore1-zetacored-persist:/root/.zetacored + + zetacore2: + volumes: + - zetacore2-zetacored-persist:/root/.zetacored + + zetacore3: + volumes: + - zetacore3-zetacored-persist:/root/.zetacored + + zetaclient0: + volumes: + - zetaclient0-zetacored-persist:/root/.zetacored + - zetaclient0-tss-persist:/root/.tss + - zetaclient0-zetaclient-persist:/root/.zetaclient + + zetaclient1: + volumes: + - zetaclient1-zetacored-persist:/root/.zetacored + - zetaclient1-tss-persist:/root/.tss + - zetaclient1-zetaclient-persist:/root/.zetaclient + + zetaclient2: + volumes: + - zetaclient2-zetacored-persist:/root/.zetacored + - zetaclient2-tss-persist:/root/.tss + - zetaclient2-zetaclient-persist:/root/.zetaclient + + zetaclient3: + volumes: + - zetaclient3-zetacored-persist:/root/.zetacored + - zetaclient3-tss-persist:/root/.tss + - zetaclient3-zetaclient-persist:/root/.zetaclient + + eth: + volumes: + - eth-data-persist:/root/data + + orchestrator: + volumes: + - orchestrator-state-persist:/root/state + +volumes: + zetacore0-zetacored-persist: + zetacore1-zetacored-persist: + zetacore2-zetacored-persist: + zetacore3-zetacored-persist: + zetaclient0-zetacored-persist: + zetaclient0-tss-persist: + zetaclient0-zetaclient-persist: + zetaclient1-zetacored-persist: + zetaclient1-tss-persist: + zetaclient1-zetaclient-persist: + zetaclient2-zetacored-persist: + zetaclient2-tss-persist: + zetaclient2-zetaclient-persist: + zetaclient3-zetacored-persist: + zetaclient3-tss-persist: + zetaclient3-zetaclient-persist: + eth-data-persist: + orchestrator-state-persist: diff --git a/contrib/localnet/docker-compose.yml b/contrib/localnet/docker-compose.yml index c25c823b8c..e3f2b5c9c9 100644 --- a/contrib/localnet/docker-compose.yml +++ b/contrib/localnet/docker-compose.yml @@ -193,7 +193,7 @@ services: networks: mynetwork: ipv4_address: 172.20.0.100 - entrypoint: ["geth", "--dev", "--http", "--http.addr", "172.20.0.100", "--http.vhosts", "*", "--http.api", "eth,web3,net", "--http.corsdomain", "https://remix.ethereum.org", "--dev.period", "2"] + entrypoint: ["geth", "--dev", "--datadir", "/root/data", "--http", "--http.addr", "172.20.0.100", "--http.vhosts", "*", "--http.api", "eth,web3,net", "--http.corsdomain", "https://remix.ethereum.org", "--dev.period", "2", "--"] eth2: build: @@ -313,7 +313,7 @@ services: - ssh:/root/.ssh grafana: - image: grafana/grafana:9.1.7 + image: grafana/grafana:10.4.8 container_name: grafana hostname: grafana profiles: @@ -322,6 +322,7 @@ services: volumes: - ./grafana/datasource.yaml:/etc/grafana/provisioning/datasources/datasource.yaml - ./grafana/dashboards/:/etc/grafana/provisioning/dashboards + - ./grafana/grafana.ini:/etc/grafana/grafana.ini - grafana_storage:/var/lib/grafana ports: - "3000:3000" diff --git a/contrib/localnet/grafana/grafana.ini b/contrib/localnet/grafana/grafana.ini new file mode 100644 index 0000000000..2542a77f90 --- /dev/null +++ b/contrib/localnet/grafana/grafana.ini @@ -0,0 +1,5 @@ +[auth.anonymous] +enabled = true + +org_id = 1 +org_role = Editor diff --git a/contrib/localnet/orchestrator/start-zetae2e.sh b/contrib/localnet/orchestrator/start-zetae2e.sh index 9e72cc15d9..a054845739 100644 --- a/contrib/localnet/orchestrator/start-zetae2e.sh +++ b/contrib/localnet/orchestrator/start-zetae2e.sh @@ -160,14 +160,18 @@ if host ton > /dev/null; then ./wait-for-ton.sh fi +# need to make the directory if it was not mounted as a volume +mkdir -p /root/state +deployed_config_path=/root/state/deployed.yml + ### Run zetae2e command depending on the option passed # Mode migrate is used to run the e2e tests before and after the TSS migration # It runs the e2e tests with the migrate flag which triggers a TSS migration at the end of the tests. Once the migrationis done the first e2e test is complete # The second e2e test is run after the migration to ensure the network is still working as expected with the new tss address if [ "$LOCALNET_MODE" == "tss-migrate" ]; then - if [[ ! -f deployed.yml ]]; then - zetae2e local $E2E_ARGS --setup-only --config config.yml --config-out deployed.yml --skip-header-proof + if [[ ! -f "$deployed_config_path" ]]; then + zetae2e local $E2E_ARGS --setup-only --config config.yml --config-out "$deployed_config_path" --skip-header-proof if [ $? -ne 0 ]; then echo "e2e setup failed" exit 1 @@ -177,7 +181,7 @@ if [ "$LOCALNET_MODE" == "tss-migrate" ]; then fi echo "running e2e test before migrating TSS" - zetae2e local $E2E_ARGS --skip-setup --config deployed.yml --skip-header-proof + zetae2e local $E2E_ARGS --skip-setup --config "$deployed_config_path" --skip-header-proof if [ $? -ne 0 ]; then echo "first e2e failed" exit 1 @@ -186,7 +190,7 @@ if [ "$LOCALNET_MODE" == "tss-migrate" ]; then echo "waiting 10 seconds for node to restart" sleep 10 - zetae2e local --skip-setup --config deployed.yml --skip-bitcoin-setup --light --skip-header-proof + zetae2e local --skip-setup --config "$deployed_config_path" --skip-bitcoin-setup --light --skip-header-proof ZETAE2E_EXIT_CODE=$? if [ $ZETAE2E_EXIT_CODE -eq 0 ]; then echo "E2E passed after migration" @@ -210,8 +214,8 @@ if [ "$LOCALNET_MODE" == "upgrade" ]; then OLD_VERSION=$(get_zetacored_version) COMMON_ARGS="--skip-header-proof --skip-tracker-check" - if [[ ! -f deployed.yml ]]; then - zetae2e local $E2E_ARGS --setup-only --config config.yml --config-out deployed.yml ${COMMON_ARGS} + if [[ ! -f "$deployed_config_path" ]]; then + zetae2e local $E2E_ARGS --setup-only --config config.yml --config-out "$deployed_config_path" ${COMMON_ARGS} if [ $? -ne 0 ]; then echo "e2e setup failed" exit 1 @@ -225,7 +229,7 @@ if [ "$LOCALNET_MODE" == "upgrade" ]; then echo "running E2E command to setup the networks and populate the state..." # Use light flag to ensure tests can complete before the upgrade height - zetae2e local $E2E_ARGS --skip-setup --config deployed.yml --light --skip-precompiles ${COMMON_ARGS} + zetae2e local $E2E_ARGS --skip-setup --config "$deployed_config_path" --light --skip-precompiles ${COMMON_ARGS} if [ $? -ne 0 ]; then echo "first e2e failed" exit 1 @@ -264,9 +268,9 @@ if [ "$LOCALNET_MODE" == "upgrade" ]; then # When the upgrade height is greater than 100 for upgrade test, the Bitcoin tests have been run once, therefore the Bitcoin wallet is already set up # Use light flag to skip advanced tests if [ "$UPGRADE_HEIGHT" -lt 100 ]; then - zetae2e local $E2E_ARGS --skip-setup --config deployed.yml --light ${COMMON_ARGS} + zetae2e local $E2E_ARGS --skip-setup --config "$deployed_config_path" --light ${COMMON_ARGS} else - zetae2e local $E2E_ARGS --skip-setup --config deployed.yml --skip-bitcoin-setup --light ${COMMON_ARGS} + zetae2e local $E2E_ARGS --skip-setup --config "$deployed_config_path" --skip-bitcoin-setup --light ${COMMON_ARGS} fi ZETAE2E_EXIT_CODE=$? @@ -282,8 +286,8 @@ else # If no mode is passed, run the e2e tests normally echo "running e2e setup..." - if [[ ! -f deployed.yml ]]; then - zetae2e local $E2E_ARGS --config config.yml --setup-only --config-out deployed.yml + if [[ ! -f "$deployed_config_path" ]]; then + zetae2e local $E2E_ARGS --config config.yml --setup-only --config-out "$deployed_config_path" if [ $? -ne 0 ]; then echo "e2e setup failed" exit 1 @@ -298,7 +302,7 @@ else echo "running e2e tests with arguments: $E2E_ARGS" - zetae2e local $E2E_ARGS --skip-setup --config deployed.yml + zetae2e local $E2E_ARGS --skip-setup --config "$deployed_config_path" ZETAE2E_EXIT_CODE=$? # if e2e passed, exit with 0, otherwise exit with 1 diff --git a/contrib/localnet/scripts/start-zetaclientd.sh b/contrib/localnet/scripts/start-zetaclientd.sh index 71ca33f589..7d2768b177 100755 --- a/contrib/localnet/scripts/start-zetaclientd.sh +++ b/contrib/localnet/scripts/start-zetaclientd.sh @@ -109,4 +109,10 @@ then fi fi +# merge extra-evm-chains.json into zetaclient_config.json if specified +if [[ -f /root/extra-evm-chains.json ]]; then + jq '.EVMChainConfigs *= input' /root/.zetacored/config/zetaclient_config.json /root/extra-evm-chains.json > /tmp/merged_config.json + mv /tmp/merged_config.json /root/.zetacored/config/zetaclient_config.json +fi + zetaclientd-supervisor start < /root/password.file \ No newline at end of file diff --git a/docs/development/LOCAL_TESTING.md b/docs/development/LOCAL_TESTING.md index 3472daca89..010cf2d8ff 100644 --- a/docs/development/LOCAL_TESTING.md +++ b/docs/development/LOCAL_TESTING.md @@ -38,6 +38,15 @@ This uses `docker compose` to start the localnet and run standard e2e tests insi - `ZETACORED_IMPORT_GENESIS_DATA`: path to genesis data to import before starting zetacored - `ZETACORED_START_PERIOD`: duration to tolerate `zetacored` health check failures during startup +More options directly to `docker compose` via the `NODE_COMPOSE_ARGS` variable. This allows setting additional profiles or configuring an overlay. Example: + +``` +example: NODE_COMPOSE_ARGS="--profile monitoring -f docker-compose-persistent.yml"` +make start-e2e-test +``` + +This starts the e2e tests while enabling the monitoring stack and persistence (data is not deleted between test runs). + #### Run admin functions e2e tests We define e2e tests allowing to test admin functionalities (emergency network pause for example).