From 562a53432a096c0bac083331539a66edff0c937c Mon Sep 17 00:00:00 2001 From: Lucas Bertrand Date: Mon, 24 Jun 2024 17:06:22 +0200 Subject: [PATCH 1/5] chore: initialize coderabbit config to focus review on code (#2374) --- .coderabbit.yaml | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) create mode 100644 .coderabbit.yaml diff --git a/.coderabbit.yaml b/.coderabbit.yaml new file mode 100644 index 0000000000..381e9d9fc4 --- /dev/null +++ b/.coderabbit.yaml @@ -0,0 +1,20 @@ +reviews: + path_filters: + - "**/*.pb.go" + - "**/*.pb.gw.go" + path_instructions: + - path: '**/*.go' + instructions: >- + Review the Go code, point out issues relative to principles of clean + code, expressiveness, and performance. + - path: '**/*.proto' + instructions: >- + Review the Protobuf definitions, point out issues relative to + compatibility, and expressiveness. + - path: '**/*.sh' + instructions: >- + Review the shell scripts, point out issues relative to security, + performance, and maintainability. + auto_review: + base_branches: + - develop \ No newline at end of file From 3eb36cde5f6de77d9386490d08d7a08bcfe4c634 Mon Sep 17 00:00:00 2001 From: Alex Gartner Date: Mon, 24 Jun 2024 09:57:38 -0700 Subject: [PATCH 2/5] refactor(localnet): reduce number of docker compose files (#2365) * refactor(localnet): reduce number of docker compose files * add ZETACORED_START_PERIOD and allow user to set E2E_ARGS * fix upgrade tests * fix upgrade-test height * rebase for start-e2e-import-mainnet-test * always use make stop-localnet * newlines and readme --- .github/workflows/build.yml | 6 +-- .github/workflows/execute_advanced_tests.yaml | 12 ++---- .github/workflows/publish-release.yml | 3 +- Makefile | 32 +++++++++++---- ...n.yml => docker-compose-additionalevm.yml} | 11 ++---- .../localnet/docker-compose-import-data.yml | 30 -------------- .../localnet/docker-compose-monitoring.yml | 2 - .../localnet/docker-compose-performance.yml | 9 ----- .../localnet/docker-compose-setup-only.yml | 9 ----- ...resstest.yml => docker-compose-stress.yml} | 39 ++++++------------- .../localnet/docker-compose-upgrade-light.yml | 11 ------ contrib/localnet/docker-compose-upgrade.yml | 14 ++----- contrib/localnet/docker-compose.yml | 16 +++++--- .../localnet/orchestrator/start-zetae2e.sh | 25 ++++++------ contrib/localnet/scripts/import-data.sh | 6 +-- .../scripts/start-upgrade-orchestrator.sh | 2 - contrib/localnet/scripts/start-zetaclientd.sh | 4 +- contrib/localnet/scripts/start-zetacored.sh | 12 +----- docs/development/LOCAL_TESTING.md | 19 +++++---- 19 files changed, 94 insertions(+), 168 deletions(-) rename contrib/localnet/{docker-compose-admin.yml => docker-compose-additionalevm.yml} (66%) delete mode 100644 contrib/localnet/docker-compose-import-data.yml delete mode 100644 contrib/localnet/docker-compose-performance.yml delete mode 100644 contrib/localnet/docker-compose-setup-only.yml rename contrib/localnet/{docker-compose-stresstest.yml => docker-compose-stress.yml} (70%) delete mode 100644 contrib/localnet/docker-compose-upgrade-light.yml diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 4fdd5c2149..fc58eb6c56 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -152,8 +152,7 @@ jobs: - name: Full Log Dump On Failure if: failure() run: | - cd contrib/localnet - docker compose logs + make stop-localnet - name: Notify Slack on Failure if: failure() && github.event_name == 'push' && github.ref == 'refs/heads/develop' @@ -167,8 +166,7 @@ jobs: - name: Stop Private Network if: always() run: | - cd contrib/localnet/ - docker compose down + make stop-localnet - name: Clean Up Workspace if: always() diff --git a/.github/workflows/execute_advanced_tests.yaml b/.github/workflows/execute_advanced_tests.yaml index e42d08e9d6..0af6d13a48 100644 --- a/.github/workflows/execute_advanced_tests.yaml +++ b/.github/workflows/execute_advanced_tests.yaml @@ -50,8 +50,7 @@ jobs: - name: Full Log Dump On Failure if: failure() run: | - cd contrib/localnet - docker compose logs + make stop-localnet - name: Notify Slack on Failure if: failure() && github.event_name == 'schedule' @@ -82,8 +81,7 @@ jobs: - name: Full Log Dump On Failure if: failure() run: | - cd contrib/localnet - docker compose logs + make stop-localnet - name: Notify Slack on Failure if: failure() && github.event_name == 'schedule' @@ -114,8 +112,7 @@ jobs: - name: Full Log Dump On Failure if: failure() run: | - cd contrib/localnet - docker compose logs + make stop-localnet e2e-performance-test: if: ${{ github.event.inputs.e2e-performance-test == 'true' }} @@ -137,5 +134,4 @@ jobs: - name: Full Log Dump On Failure if: failure() run: | - cd contrib/localnet - docker compose logs \ No newline at end of file + make stop-localnet diff --git a/.github/workflows/publish-release.yml b/.github/workflows/publish-release.yml index e96bb39e46..89a4b1f688 100644 --- a/.github/workflows/publish-release.yml +++ b/.github/workflows/publish-release.yml @@ -339,8 +339,7 @@ jobs: - name: Stop Private Network if: ${{ always() && github.event.inputs.skip_checks != 'true' }} run: | - cd contrib/localnet/ - docker compose down + make stop-localnet - name: Clean Up Workspace if: always() diff --git a/Makefile b/Makefile index 360e071485..75834d690d 100644 --- a/Makefile +++ b/Makefile @@ -26,6 +26,14 @@ HSM_BUILD_FLAGS := -tags pebbledb,ledger,hsm_test export DOCKER_BUILDKIT := 1 +# parameters for localnet docker compose files +# set defaults to empty to prevent docker warning +export LOCALNET_MODE +export E2E_ARGS := $(E2E_ARGS) +export UPGRADE_HEIGHT +export ZETACORED_IMPORT_GENESIS_DATA +export ZETACORED_START_PERIOD := 30s + clean: clean-binaries clean-dir clean-test-dir clean-coverage clean-binaries: @@ -217,15 +225,17 @@ start-e2e-test: zetanode start-e2e-admin-test: zetanode @echo "--> Starting e2e admin test" - cd contrib/localnet/ && $(DOCKER) compose -f docker-compose.yml -f docker-compose-admin.yml up -d + export E2E_ARGS="--skip-regular --test-admin" && \ + cd contrib/localnet/ && $(DOCKER) compose -f docker-compose.yml -f docker-compose-additionalevm.yml up -d start-e2e-performance-test: zetanode @echo "--> Starting e2e performance test" - cd contrib/localnet/ && $(DOCKER) compose -f docker-compose.yml -f docker-compose-performance.yml up -d + export E2E_ARGS="--test-performance" && \ + cd contrib/localnet/ && $(DOCKER) compose -f docker-compose.yml up -d start-stress-test: zetanode @echo "--> Starting stress test" - cd contrib/localnet/ && $(DOCKER) compose -f docker-compose.yml -f docker-compose-stresstest.yml up -d + cd contrib/localnet/ && $(DOCKER) compose -f docker-compose.yml -f docker-compose-stress.yml up -d #TODO: replace OLD_VERSION with v16 tag once its available zetanode-upgrade: zetanode @@ -236,24 +246,30 @@ zetanode-upgrade: zetanode start-upgrade-test: zetanode-upgrade @echo "--> Starting upgrade test" + export LOCALNET_MODE=upgrade && \ + export UPGRADE_HEIGHT=225 && \ cd contrib/localnet/ && $(DOCKER) compose -f docker-compose.yml -f docker-compose-upgrade.yml up -d start-upgrade-test-light: zetanode-upgrade @echo "--> Starting light upgrade test (no ZetaChain state populating before upgrade)" - cd contrib/localnet/ && $(DOCKER) compose -f docker-compose.yml -f docker-compose-upgrade.yml -f docker-compose-upgrade-light.yml up -d + export LOCALNET_MODE=upgrade && \ + export UPGRADE_HEIGHT=90 && \ + cd contrib/localnet/ && $(DOCKER) compose -f docker-compose.yml -f docker-compose-upgrade.yml up -d start-localnet: zetanode start-localnet-skip-build start-localnet-skip-build: @echo "--> Starting localnet" - cd contrib/localnet/ && $(DOCKER) compose -f docker-compose.yml -f docker-compose-setup-only.yml up -d + export LOCALNET_MODE=setup-only && \ + cd contrib/localnet/ && $(DOCKER) compose -f docker-compose.yml up -d -stop-localnet: start-e2e-import-mainnet-test: zetanode @echo "--> Starting e2e import-data test" - cd contrib/localnet/ && ./scripts/import-data.sh mainnet && $(DOCKER) compose -f docker-compose.yml -f docker-compose-import-data.yml up -d + export ZETACORED_IMPORT_GENESIS_DATA=true && \ + export ZETACORED_START_PERIOD=10m && \ + cd contrib/localnet/ && ./scripts/import-data.sh mainnet && $(DOCKER) compose -f docker-compose.yml up -d -stop-test: +stop-localnet: cd contrib/localnet/ && $(DOCKER) compose down --remove-orphans ############################################################################### diff --git a/contrib/localnet/docker-compose-admin.yml b/contrib/localnet/docker-compose-additionalevm.yml similarity index 66% rename from contrib/localnet/docker-compose-admin.yml rename to contrib/localnet/docker-compose-additionalevm.yml index 19a478100e..208a8e30e9 100644 --- a/contrib/localnet/docker-compose-admin.yml +++ b/contrib/localnet/docker-compose-additionalevm.yml @@ -1,13 +1,8 @@ -version: "3" - # This docker-compose file overrides the orchestrator service to specify the flag to test the admin functions # and skip the regular tests # it also adds another local Ethereum network to test EVM chain migration and use the additional-evm flag services: - orchestrator: - entrypoint: ["/work/start-zetae2e.sh", "local --skip-regular --test-admin"] - eth2: build: context: ./anvil @@ -21,7 +16,9 @@ services: ipv4_address: 172.20.0.102 zetaclient0: - entrypoint: [ "/root/start-zetaclientd.sh", "additional-evm" ] + environment: + - ADDITIONAL_EVM=true zetaclient1: - entrypoint: [ "/root/start-zetaclientd.sh", "additional-evm" ] + environment: + - ADDITIONAL_EVM=true diff --git a/contrib/localnet/docker-compose-import-data.yml b/contrib/localnet/docker-compose-import-data.yml deleted file mode 100644 index 19f4068ad5..0000000000 --- a/contrib/localnet/docker-compose-import-data.yml +++ /dev/null @@ -1,30 +0,0 @@ -version: "3" - -# This docker-compose file configures the localnet environment -# it contains the following services: -# - ZetaChain with 2 nodes (zetacore0, zetacore1) -# - A observer set with 2 clients (zetaclient0, zetaclient1) -# - An Ethereum node (eth) -# - A Bitcoin node (bitcoin) -# - A Rosetta API (rosetta) -# - An orchestrator to manage interaction with the localnet (orchestrator) -services: - rosetta: - entrypoint: ["/root/start-rosetta.sh"] - - zetacore0: - entrypoint: ["/root/start-zetacored.sh", "2","import-data"] - volumes: - - ~/genesis_export/:/root/genesis_data - - zetacore1: - entrypoint: ["/root/start-zetacored.sh", "2","import-data"] - - zetaclient0: - entrypoint: ["/root/start-zetaclientd.sh"] - - zetaclient1: - entrypoint: ["/root/start-zetaclientd.sh"] - - orchestrator: - entrypoint: ["/work/start-zetae2e.sh", "local"] \ No newline at end of file diff --git a/contrib/localnet/docker-compose-monitoring.yml b/contrib/localnet/docker-compose-monitoring.yml index ad66f828ec..9f11bd5575 100644 --- a/contrib/localnet/docker-compose-monitoring.yml +++ b/contrib/localnet/docker-compose-monitoring.yml @@ -1,5 +1,3 @@ -version: '3' - services: grafana: image: grafana/grafana:latest diff --git a/contrib/localnet/docker-compose-performance.yml b/contrib/localnet/docker-compose-performance.yml deleted file mode 100644 index 0374d42030..0000000000 --- a/contrib/localnet/docker-compose-performance.yml +++ /dev/null @@ -1,9 +0,0 @@ -version: "3" - -# This docker-compose file overrides the orchestrator service to specify the flag to test performance of cctxs -# and skip the regular tests - -services: - orchestrator: - entrypoint: ["/work/start-zetae2e.sh", "local --test-performance"] - diff --git a/contrib/localnet/docker-compose-setup-only.yml b/contrib/localnet/docker-compose-setup-only.yml deleted file mode 100644 index 11761ad817..0000000000 --- a/contrib/localnet/docker-compose-setup-only.yml +++ /dev/null @@ -1,9 +0,0 @@ -version: "3" - -# This docker-compose file overrides the orcherstrator service to use the setup only option -# It is used to setup the localnet environment without running tests - -services: - orchestrator: - entrypoint: ["/work/start-zetae2e.sh", "local --setup-only"] - diff --git a/contrib/localnet/docker-compose-stresstest.yml b/contrib/localnet/docker-compose-stress.yml similarity index 70% rename from contrib/localnet/docker-compose-stresstest.yml rename to contrib/localnet/docker-compose-stress.yml index 38df1133f3..fb11f01b5c 100644 --- a/contrib/localnet/docker-compose-stresstest.yml +++ b/contrib/localnet/docker-compose-stress.yml @@ -1,23 +1,19 @@ -version: "3" - # This docker-compose redefine the services: # - ZetaChain with 4 nodes to test performance # - ZetaChain observer set with 4 clients to test performance -# - Orchestrator call stress commands services: zetacore0: - entrypoint: ["/root/start-zetacored.sh", "4"] + environment: + - ZETACORED_REPLICAS=4 zetacore1: - entrypoint: ["/root/start-zetacored.sh", "4"] + environment: + - ZETACORED_REPLICAS=4 zetacore2: image: zetanode:latest container_name: zetacore2 - build: - context: ../../. - dockerfile: Dockerfile-localnet hostname: zetacore2 networks: mynetwork: @@ -26,6 +22,7 @@ services: environment: - HOTKEY_BACKEND=file - HOTKEY_PASSWORD=password # test purposes only + - ZETACORED_REPLICAS=4 zetacore3: image: zetanode:latest @@ -41,21 +38,11 @@ services: environment: - HOTKEY_BACKEND=file - HOTKEY_PASSWORD=password # test purposes only - - zetaclient0: - ports: - - "8123:8123" - entrypoint: /root/start-zetaclientd.sh - - zetaclient1: - entrypoint: /root/start-zetaclientd.sh + - ZETACORED_REPLICAS=4 zetaclient2: image: zetanode:latest container_name: zetaclient2 - build: - context: ../../. - dockerfile: Dockerfile-localnet hostname: zetaclient2 networks: mynetwork: @@ -65,13 +52,13 @@ services: - ETHDEV_ENDPOINT=http://eth:8545 - HOTKEY_BACKEND=file - HOTKEY_PASSWORD=password # test purposes only + volumes: + - ssh:/root/.ssh + - preparams:/root/preparams zetaclient3: image: zetanode:latest container_name: zetaclient3 - build: - context: ../../. - dockerfile: Dockerfile-localnet hostname: zetaclient3 networks: mynetwork: @@ -81,8 +68,6 @@ services: - ETHDEV_ENDPOINT=http://eth:8545 - HOTKEY_BACKEND=file - HOTKEY_PASSWORD=password # test purposes only - - orchestrator: - build: - dockerfile: contrib/localnet/orchestrator/Dockerfile.fastbuild - entrypoint: ["/work/start-zetae2e.sh", "stress"] \ No newline at end of file + volumes: + - ssh:/root/.ssh + - preparams:/root/preparams diff --git a/contrib/localnet/docker-compose-upgrade-light.yml b/contrib/localnet/docker-compose-upgrade-light.yml deleted file mode 100644 index 8e3cb098c1..0000000000 --- a/contrib/localnet/docker-compose-upgrade-light.yml +++ /dev/null @@ -1,11 +0,0 @@ -version: "3" - -# This docker-compose is similar to the docker-compose-upgrade.yml, but it uses a smaller height option for the upgrade (90) -# By using 90, the orchestrator will automatically run setup only for the first e2e test execution. - -services: - orchestrator: - entrypoint: ["/work/start-zetae2e.sh", "local", "upgrade", "90"] - - upgrade-orchestrator: - entrypoint: ["/root/start-upgrade-orchestrator.sh", "90"] \ No newline at end of file diff --git a/contrib/localnet/docker-compose-upgrade.yml b/contrib/localnet/docker-compose-upgrade.yml index a7497db980..c6ae72941a 100644 --- a/contrib/localnet/docker-compose-upgrade.yml +++ b/contrib/localnet/docker-compose-upgrade.yml @@ -1,5 +1,3 @@ -version: "3" - # This docker-compose redefine the services: # - ZetaChain with 2 nodes (zetacore0, zetacore1) using the upgrade option for cosmovisor # - ZetaChain observer set with 2 clients (zetaclient0, zetaclient1) using the background option @@ -13,17 +11,11 @@ services: image: zetanode:old zetaclient0: - entrypoint: ["/root/start-zetaclientd.sh"] image: zetanode:old zetaclient1: - entrypoint: ["/root/start-zetaclientd.sh"] image: zetanode:old - orchestrator: - entrypoint: ["/work/start-zetae2e.sh", "local", "upgrade"] - image: orchestrator - upgrade-host: image: zetanode:latest container_name: upgrade-host @@ -40,12 +32,14 @@ services: image: zetanode:old container_name: upgrade-orchestrator hostname: upgrade-orchestrator - entrypoint: ["/root/start-upgrade-orchestrator.sh", "225"] + entrypoint: ["/root/start-upgrade-orchestrator.sh"] networks: mynetwork: ipv4_address: 172.20.0.251 depends_on: - zetacore0 - upgrade-host + environment: + - UPGRADE_HEIGHT=${UPGRADE_HEIGHT} volumes: - - ssh:/root/.ssh \ No newline at end of file + - ssh:/root/.ssh diff --git a/contrib/localnet/docker-compose.yml b/contrib/localnet/docker-compose.yml index f176c30f00..1a59202d0d 100644 --- a/contrib/localnet/docker-compose.yml +++ b/contrib/localnet/docker-compose.yml @@ -1,5 +1,3 @@ -version: "3" - # This docker-compose file configures the localnet environment # it contains the following services: # - ZetaChain with 2 nodes (zetacore0, zetacore1) @@ -46,17 +44,20 @@ services: interval: 30s timeout: 10s retries: 3 - start_period: 30s + start_period: ${ZETACORED_START_PERIOD} start_interval: 1s networks: mynetwork: ipv4_address: 172.20.0.11 - entrypoint: ["/root/start-zetacored.sh", "2"] + entrypoint: ["/root/start-zetacored.sh"] environment: - HOTKEY_BACKEND=file - HOTKEY_PASSWORD=password # test purposes only + - ZETACORED_REPLICAS=2 + - ZETACORED_IMPORT_GENESIS_DATA=${ZETACORED_IMPORT_GENESIS_DATA} volumes: - ssh:/root/.ssh + - ~/.zetacored/genesis_data:/root/genesis_data zetacore1: image: zetanode:latest @@ -65,10 +66,11 @@ services: networks: mynetwork: ipv4_address: 172.20.0.12 - entrypoint: ["/root/start-zetacored.sh", "2"] + entrypoint: ["/root/start-zetacored.sh"] environment: - HOTKEY_BACKEND=file - HOTKEY_PASSWORD=password # test purposes only + - ZETACORED_REPLICAS=2 volumes: - ssh:/root/.ssh @@ -146,6 +148,10 @@ services: mynetwork: ipv4_address: 172.20.0.2 entrypoint: ["/work/start-zetae2e.sh", "local"] + environment: + - LOCALNET_MODE=${LOCALNET_MODE} + - E2E_ARGS=${E2E_ARGS} + - UPGRADE_HEIGHT=${UPGRADE_HEIGHT} volumes: - ssh:/root/.ssh volumes: diff --git a/contrib/localnet/orchestrator/start-zetae2e.sh b/contrib/localnet/orchestrator/start-zetae2e.sh index 8dcb4ea271..791f0a6074 100644 --- a/contrib/localnet/orchestrator/start-zetae2e.sh +++ b/contrib/localnet/orchestrator/start-zetae2e.sh @@ -5,9 +5,6 @@ # A second optional argument can be passed and can have the following value: # upgrade: run the local e2e once, then restart zetaclientd at upgrade height and run the local e2e again -ZETAE2E_CMD=$1 -OPTION=$2 - get_zetacored_version() { retries=10 node_info="" @@ -78,15 +75,15 @@ geth --exec 'eth.sendTransaction({from: eth.coinbase, to: "0xF421292cb0d3c97b90E ### Run zetae2e command depending on the option passed -if [ "$OPTION" == "upgrade" ]; then +if [ "$LOCALNET_MODE" == "upgrade" ]; then # Run the e2e tests, then restart zetaclientd at upgrade height and run the e2e tests again - # Fetch the height of the upgrade, default is 225, if arg3 is passed, use that value - UPGRADE_HEIGHT=${3:-225} + # set upgrade height to 225 by default + UPGRADE_HEIGHT=${UPGRADE_HEIGHT:=225} if [[ ! -f deployed.yml ]]; then - zetae2e "$ZETAE2E_CMD" --setup-only --config-out deployed.yml --skip-header-proof + zetae2e local $E2E_ARGS --setup-only --config-out deployed.yml --skip-header-proof if [ $? -ne 0 ]; then echo "e2e setup failed" exit 1 @@ -100,7 +97,7 @@ if [ "$OPTION" == "upgrade" ]; then echo "running E2E command to setup the networks and populate the state..." # Use light flag to ensure tests can complete before the upgrade height - zetae2e "$ZETAE2E_CMD" --skip-setup --config deployed.yml --light --skip-header-proof + zetae2e local $E2E_ARGS --skip-setup --config deployed.yml --light --skip-header-proof if [ $? -ne 0 ]; then echo "first e2e failed" exit 1 @@ -142,9 +139,9 @@ if [ "$OPTION" == "upgrade" ]; then # When the upgrade height is greater than 100 for upgrade test, the Bitcoin tests have been run once, therefore the Bitcoin wallet is already set up # Use light flag to skip advanced tests if [ "$UPGRADE_HEIGHT" -lt 100 ]; then - zetae2e $ZETAE2E_CMD --skip-setup --config deployed.yml --light --skip-header-proof + zetae2e local $E2E_ARGS --skip-setup --config deployed.yml --light --skip-header-proof else - zetae2e $ZETAE2E_CMD --skip-setup --config deployed.yml --skip-bitcoin-setup --light --skip-header-proof + zetae2e local $E2E_ARGS --skip-setup --config deployed.yml --skip-bitcoin-setup --light --skip-header-proof fi ZETAE2E_EXIT_CODE=$? @@ -162,7 +159,7 @@ else echo "running e2e setup..." if [[ ! -f deployed.yml ]]; then - zetae2e $ZETAE2E_CMD --setup-only --config-out deployed.yml + zetae2e local $E2E_ARGS --setup-only --config-out deployed.yml if [ $? -ne 0 ]; then echo "e2e setup failed" exit 1 @@ -171,9 +168,13 @@ else echo "skipping e2e setup because it has already been completed" fi + if [ "$LOCALNET_MODE" == "setup-only" ]; then + exit 0 + fi + echo "running e2e tests..." - zetae2e $ZETAE2E_CMD --skip-setup --config deployed.yml + zetae2e local $E2E_ARGS --skip-setup --config deployed.yml ZETAE2E_EXIT_CODE=$? # if e2e passed, exit with 0, otherwise exit with 1 diff --git a/contrib/localnet/scripts/import-data.sh b/contrib/localnet/scripts/import-data.sh index d71d5c3656..5236bfe166 100644 --- a/contrib/localnet/scripts/import-data.sh +++ b/contrib/localnet/scripts/import-data.sh @@ -7,9 +7,9 @@ fi NETWORK=$1 echo "NETWORK: ${NETWORK}" -rm -rf ~/genesis_export/ -mkdir ~/genesis_export/ +rm -rf ~/.zetacored/genesis_data +mkdir ~/.zetacored/genesis_data echo "Download Latest State Export" LATEST_EXPORT_URL=$(curl https://snapshots.zetachain.com/latest-state-export | jq -r ."${NETWORK}") echo "LATEST EXPORT URL: ${LATEST_EXPORT_URL}" -wget -q ${LATEST_EXPORT_URL} -O ~/genesis_export/exported-genesis.json \ No newline at end of file +wget -q ${LATEST_EXPORT_URL} -O ~/.zetacored/genesis_data/exported-genesis.json diff --git a/contrib/localnet/scripts/start-upgrade-orchestrator.sh b/contrib/localnet/scripts/start-upgrade-orchestrator.sh index a2e8ec68fd..f68009db9a 100755 --- a/contrib/localnet/scripts/start-upgrade-orchestrator.sh +++ b/contrib/localnet/scripts/start-upgrade-orchestrator.sh @@ -1,7 +1,5 @@ #!/bin/bash -UPGRADE_HEIGHT=$1 - CHAINID="athens_101-1" UPGRADE_AUTHORITY_ACCOUNT="zeta10d07y265gmmuvt4z0w9aw880jnsr700jvxasvr" diff --git a/contrib/localnet/scripts/start-zetaclientd.sh b/contrib/localnet/scripts/start-zetaclientd.sh index 1b018de4ef..a849891275 100755 --- a/contrib/localnet/scripts/start-zetaclientd.sh +++ b/contrib/localnet/scripts/start-zetaclientd.sh @@ -63,7 +63,7 @@ then # check if the option is additional-evm # in this case, the additional evm is represented with the sepolia chain, we set manually the eth2 endpoint to the sepolia chain (11155111 -> http://eth2:8545) # in /root/.zetacored/config/zetaclient_config.json - if [ "$OPTION" == "additional-evm" ]; then + if [[ -n $ADDITIONAL_EVM ]]; then set_sepolia_endpoint fi fi @@ -82,7 +82,7 @@ then # check if the option is additional-evm # in this case, the additional evm is represented with the sepolia chain, we set manually the eth2 endpoint to the sepolia chain (11155111 -> http://eth2:8545) # in /root/.zetacored/config/zetaclient_config.json - if [ "$OPTION" == "additional-evm" ]; then + if [[ -n $ADDITIONAL_EVM ]]; then set_sepolia_endpoint fi fi diff --git a/contrib/localnet/scripts/start-zetacored.sh b/contrib/localnet/scripts/start-zetacored.sh index d390b99083..dd8509847f 100755 --- a/contrib/localnet/scripts/start-zetacored.sh +++ b/contrib/localnet/scripts/start-zetacored.sh @@ -67,14 +67,6 @@ add_v17_message_authorizations() { ' $json_file > temp.json && mv temp.json $json_file } -if [ $# -lt 1 ] -then - echo "Usage: genesis.sh [option]" - exit 1 -fi -NUMOFNODES=$1 -OPTION=$2 - # create keys CHAINID="athens_101-1" KEYRING="test" @@ -96,7 +88,7 @@ export UNSAFE_SKIP_BACKUP=true # generate node list START=1 # shellcheck disable=SC2100 -END=$((NUMOFNODES - 1)) +END=$((ZETACORED_REPLICAS - 1)) NODELIST=() for i in $(eval echo "{$START..$END}") do @@ -257,7 +249,7 @@ then scp $NODE:~/.zetacored/config/gentx/* ~/.zetacored/config/gentx/z2gentx/ done - if [[ "$OPTION" == "import-data" || "$OPTION" == "import-data-upgrade" ]]; then + if [[ -n "$ZETACORED_IMPORT_GENESIS_DATA" ]]; then echo "Importing data" zetacored parse-genesis-file /root/genesis_data/exported-genesis.json fi diff --git a/docs/development/LOCAL_TESTING.md b/docs/development/LOCAL_TESTING.md index d28c44638e..3472daca89 100644 --- a/docs/development/LOCAL_TESTING.md +++ b/docs/development/LOCAL_TESTING.md @@ -27,6 +27,17 @@ Now we have built all the docker images, we can run the e2e test with make comma make start-e2e-test ``` +This uses `docker compose` to start the localnet and run standard e2e tests inside the orchestrator container. There are several parameters that the `Makefile` can provide to `docker compose` via environment variables: + +- `LOCALNET_MODE` + - `setup-only`: only setup the localnet, do not run the e2e tests + - `upgrade`: run the upgrade tests + - unset: run the e2e tests +- `E2E_ARGS`: arguments to provide to the `zetae2e local` command +- `UPGRADE_HEIGHT`: block height to upgrade at when `LOCALNET_MODE=upgrade` +- `ZETACORED_IMPORT_GENESIS_DATA`: path to genesis data to import before starting zetacored +- `ZETACORED_START_PERIOD`: duration to tolerate `zetacored` health check failures during startup + #### Run admin functions e2e tests We define e2e tests allowing to test admin functionalities (emergency network pause for example). @@ -42,13 +53,7 @@ make start-e2e-admin-test Upgrade tests run the E2E tests with an older version, upgrade the nodes to the new version, and run the E2E tests again. This allows testing the upgrade process with a populated state. -Before running the upgrade tests, the versions must be specified in `Dockefile-upgrade`: - -```dockerfile -ARG OLD_VERSION=v{old_major_version}.{old_minor_version}.{old_patch_version} -ENV NEW_VERSION=v{new_major_version} -``` -The new version must match the version specified in `app/setup_handlers.go` +Before running the upgrade tests, the old version must be specified the Makefile. NOTE: We only specify the major version for `NEW_VERSION` since we use major version only for chain upgrade. Semver is needed for `OLD_VERSION` because we use this value to fetch the release tag from the GitHub repository. From 3a96ba6ca0342d24734373e99b260f3a195a2a4d Mon Sep 17 00:00:00 2001 From: Lucas Bertrand Date: Mon, 24 Jun 2024 20:55:13 +0200 Subject: [PATCH 3/5] fix: use correct name for rate limiter flag query (#2367) * fix: use correct name for rate limiter flag query * make generate * Update x/crosschain/client/cli/query_rate_limiter_flags.go Co-authored-by: Tanmay * make generate --------- Co-authored-by: Tanmay --- docs/cli/zetacored/zetacored_query_crosschain.md | 2 +- ...> zetacored_query_crosschain_show-rate-limiter-flags.md} | 6 +++--- x/crosschain/client/cli/query_rate_limiter_flags.go | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) rename docs/cli/zetacored/{zetacored_query_crosschain_update_rate_limit_flags.md => zetacored_query_crosschain_show-rate-limiter-flags.md} (86%) diff --git a/docs/cli/zetacored/zetacored_query_crosschain.md b/docs/cli/zetacored/zetacored_query_crosschain.md index 595a065ea5..6ee04cc8d2 100644 --- a/docs/cli/zetacored/zetacored_query_crosschain.md +++ b/docs/cli/zetacored/zetacored_query_crosschain.md @@ -41,5 +41,5 @@ zetacored query crosschain [flags] * [zetacored query crosschain show-gas-price](zetacored_query_crosschain_show-gas-price.md) - shows a gasPrice * [zetacored query crosschain show-inbound-hash-to-cctx](zetacored_query_crosschain_show-inbound-hash-to-cctx.md) - shows a inboundHashToCctx * [zetacored query crosschain show-outbound-tracker](zetacored_query_crosschain_show-outbound-tracker.md) - shows an outbound tracker -* [zetacored query crosschain update_rate_limit_flags](zetacored_query_crosschain_update_rate_limit_flags.md) - shows the rate limiter flags +* [zetacored query crosschain show-rate-limiter-flags](zetacored_query_crosschain_show-rate-limiter-flags.md) - shows the rate limiter flags diff --git a/docs/cli/zetacored/zetacored_query_crosschain_update_rate_limit_flags.md b/docs/cli/zetacored/zetacored_query_crosschain_show-rate-limiter-flags.md similarity index 86% rename from docs/cli/zetacored/zetacored_query_crosschain_update_rate_limit_flags.md rename to docs/cli/zetacored/zetacored_query_crosschain_show-rate-limiter-flags.md index 6dcc3b12a4..abbf7042bb 100644 --- a/docs/cli/zetacored/zetacored_query_crosschain_update_rate_limit_flags.md +++ b/docs/cli/zetacored/zetacored_query_crosschain_show-rate-limiter-flags.md @@ -1,9 +1,9 @@ -# query crosschain update_rate_limit_flags +# query crosschain show-rate-limiter-flags shows the rate limiter flags ``` -zetacored query crosschain update_rate_limit_flags [flags] +zetacored query crosschain show-rate-limiter-flags [flags] ``` ### Options @@ -12,7 +12,7 @@ zetacored query crosschain update_rate_limit_flags [flags] --grpc-addr string the gRPC endpoint to use for this chain --grpc-insecure allow gRPC over insecure channels, if not TLS the server must use TLS --height int Use a specific height to query state at (this can error if the node is pruning state) - -h, --help help for update_rate_limit_flags + -h, --help help for show-rate-limiter-flags --node string [host]:[port] to Tendermint RPC interface for this chain -o, --output string Output format (text|json) ``` diff --git a/x/crosschain/client/cli/query_rate_limiter_flags.go b/x/crosschain/client/cli/query_rate_limiter_flags.go index ad4177295e..bdd8fd0558 100644 --- a/x/crosschain/client/cli/query_rate_limiter_flags.go +++ b/x/crosschain/client/cli/query_rate_limiter_flags.go @@ -12,7 +12,7 @@ import ( func CmdShowUpdateRateLimiterFlags() *cobra.Command { cmd := &cobra.Command{ - Use: "update_rate_limit_flags", + Use: "show-rate-limiter-flags", Short: "shows the rate limiter flags", RunE: func(cmd *cobra.Command, _ []string) error { clientCtx := client.GetClientContextFromCmd(cmd) From a1af6298e4b9c923dbac1371275b9816118281ae Mon Sep 17 00:00:00 2001 From: Christopher Fuka <97121270+CryptoFewka@users.noreply.github.com> Date: Mon, 24 Jun 2024 15:31:44 -0500 Subject: [PATCH 4/5] ci: Update sast-linters.yml (#2377) * Update sast-linters.yml - Exclude .github/workflows/sast-linters.yml from gosec nosec use scanning * Update sast-linters.yml * Update changelog.md * Update sast-linters.yml Also ignore changelog.md --- .github/workflows/sast-linters.yml | 13 +++++++++---- changelog.md | 3 ++- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/.github/workflows/sast-linters.yml b/.github/workflows/sast-linters.yml index 2372ff5264..79b308d2df 100644 --- a/.github/workflows/sast-linters.yml +++ b/.github/workflows/sast-linters.yml @@ -104,7 +104,14 @@ jobs: echo "Changed files: ${{ steps.changed-files.outputs.all_changed_files }}" for file in ${{ steps.changed-files.outputs.all_changed_files }}; do - if git diff ${{ github.event.pull_request.base.sha }} $file | grep -q nosec; then + # Skip this workflow file + if [ "$file" == ".github/workflows/sast-linters.yml" ] || [ "$file" == "changelog.md" ]; then + echo "Skipping nosec check for $file" + continue + fi + + # Only consider additions of "nosec", marked by '+' + if git diff ${{ github.event.pull_request.base.sha }} $file | grep -q '^+.*nosec'; then echo "nosec detected in $file" nosec_list+=("$file,") nosec_detected=1 @@ -118,6 +125,7 @@ jobs: echo "nosec_files=$nosec_list_string" >> $GITHUB_ENV echo "nosec_detected=$nosec_detected" >> $GITHUB_ENV + - name: Report nosec uses uses: mshick/add-pr-comment@v2 if: env.nosec_detected == 1 @@ -149,6 +157,3 @@ jobs: run: | DIFF=$(git diff ${{ github.event.pull_request.base.sha }}) echo "$DIFF" | grep -P '#nosec(?!(\sG\d{3}))(?![^\s\t])([\s\t]*|$)' && echo "nosec without specified rule found!" && exit 1 || exit 0 - - - \ No newline at end of file diff --git a/changelog.md b/changelog.md index f7a819e383..9d439de407 100644 --- a/changelog.md +++ b/changelog.md @@ -92,6 +92,7 @@ * [2191](https://github.com/zeta-chain/node/pull/2191) - Fixed conditional logic for the docker build step for non release builds to not overwrite the github tag * [2192](https://github.com/zeta-chain/node/pull/2192) - Added release status checker and updater pipeline that will update release statuses when they go live on network * [2335](https://github.com/zeta-chain/node/pull/2335) - ci: updated the artillery report to publish to artillery cloud +* [2377](https://github.com/zeta-chain/node/pull/2377) - ci: adjusted sast-linters.yml to not scan itself, nor alert on removal of nosec. ## v17.0.0 @@ -531,4 +532,4 @@ Getting the correct TSS address for Bitcoin now requires proviidng the Bitcoin c ### CI * [1218](https://github.com/zeta-chain/node/pull/1218) - cross-compile release binaries and simplify PR testings -* [1302](https://github.com/zeta-chain/node/pull/1302) - add mainnet builds to goreleaser \ No newline at end of file +* [1302](https://github.com/zeta-chain/node/pull/1302) - add mainnet builds to goreleaser From 852897fdca9b497190d225e77994b516cc77dbb0 Mon Sep 17 00:00:00 2001 From: Tanmay Date: Mon, 24 Jun 2024 17:59:30 -0400 Subject: [PATCH 5/5] feat: add migration for authorization list (#2366) * add migration for authorization list * add changelog * add forced migration to fix upgrade tests * make fmt --------- Co-authored-by: Alex Gartner --- Makefile | 2 +- app/setup_handlers.go | 2 ++ changelog.md | 1 + x/authority/keeper/migrator.go | 24 +++++++++++++++++ x/authority/migrations/v2/migrate.go | 21 +++++++++++++++ x/authority/migrations/v2/migrate_test.go | 28 ++++++++++++++++++++ x/authority/module.go | 6 ++++- x/authority/types/authorization_list_test.go | 5 ++++ 8 files changed, 87 insertions(+), 2 deletions(-) create mode 100644 x/authority/keeper/migrator.go create mode 100644 x/authority/migrations/v2/migrate.go create mode 100644 x/authority/migrations/v2/migrate_test.go diff --git a/Makefile b/Makefile index 75834d690d..858efad24a 100644 --- a/Makefile +++ b/Makefile @@ -240,7 +240,7 @@ start-stress-test: zetanode #TODO: replace OLD_VERSION with v16 tag once its available zetanode-upgrade: zetanode @echo "Building zetanode-upgrade" - $(DOCKER) build -t zetanode:old -f Dockerfile-localnet --target old-runtime --build-arg OLD_VERSION='release/v16' . + $(DOCKER) build -t zetanode:old -f Dockerfile-localnet --target old-runtime --build-arg OLD_VERSION='release/v17' . $(DOCKER) build -t orchestrator -f contrib/localnet/orchestrator/Dockerfile.fastbuild . .PHONY: zetanode-upgrade diff --git a/app/setup_handlers.go b/app/setup_handlers.go index 7fe7bc1f8f..df6ef31c9b 100644 --- a/app/setup_handlers.go +++ b/app/setup_handlers.go @@ -24,6 +24,7 @@ import ( "golang.org/x/exp/slices" "github.com/zeta-chain/zetacore/pkg/constant" + authoritytypes "github.com/zeta-chain/zetacore/x/authority/types" emissionstypes "github.com/zeta-chain/zetacore/x/emissions/types" ibccrosschaintypes "github.com/zeta-chain/zetacore/x/ibccrosschain/types" ) @@ -66,6 +67,7 @@ func SetupHandlers(app *App) { govtypes.ModuleName, crisistypes.ModuleName, emissionstypes.ModuleName, + authoritytypes.ModuleName, } allUpgrades := upgradeTracker{ upgrades: []upgradeTrackerItem{ diff --git a/changelog.md b/changelog.md index 9d439de407..70116d7939 100644 --- a/changelog.md +++ b/changelog.md @@ -28,6 +28,7 @@ * [2319](https://github.com/zeta-chain/node/pull/2319) - use `CheckAuthorization` function in all messages * [2325](https://github.com/zeta-chain/node/pull/2325) - revert telemetry server changes * [2339](https://github.com/zeta-chain/node/pull/2339) - add binaries related question to syncing issue form +* [2366](https://github.com/zeta-chain/node/pull/2366) - add migration script for adding authorizations table ### Refactor diff --git a/x/authority/keeper/migrator.go b/x/authority/keeper/migrator.go new file mode 100644 index 0000000000..9edf014b1d --- /dev/null +++ b/x/authority/keeper/migrator.go @@ -0,0 +1,24 @@ +package keeper + +import ( + sdk "github.com/cosmos/cosmos-sdk/types" + + v2 "github.com/zeta-chain/zetacore/x/authority/migrations/v2" +) + +// Migrator is a struct for handling in-place store migrations. +type Migrator struct { + authorityKeeper Keeper +} + +// NewMigrator returns a new Migrator for the authority module. +func NewMigrator(keeper Keeper) Migrator { + return Migrator{ + authorityKeeper: keeper, + } +} + +// Migrate1to2 migrates the authority store from consensus version 1 to 2 +func (m Migrator) Migrate1to2(ctx sdk.Context) error { + return v2.MigrateStore(ctx, m.authorityKeeper) +} diff --git a/x/authority/migrations/v2/migrate.go b/x/authority/migrations/v2/migrate.go new file mode 100644 index 0000000000..5b9bbd3d83 --- /dev/null +++ b/x/authority/migrations/v2/migrate.go @@ -0,0 +1,21 @@ +package v2 + +import ( + sdk "github.com/cosmos/cosmos-sdk/types" + + "github.com/zeta-chain/zetacore/x/authority/types" +) + +type authorityKeeper interface { + SetAuthorizationList(ctx sdk.Context, list types.AuthorizationList) +} + +// MigrateStore migrates the authority module state from the consensus version 1 to 2 +func MigrateStore( + ctx sdk.Context, + keeper authorityKeeper, +) error { + ctx.Logger().Info("Migrating authority store from version 1 to 2") + keeper.SetAuthorizationList(ctx, types.DefaultAuthorizationsList()) + return nil +} diff --git a/x/authority/migrations/v2/migrate_test.go b/x/authority/migrations/v2/migrate_test.go new file mode 100644 index 0000000000..d8cbdbf6fe --- /dev/null +++ b/x/authority/migrations/v2/migrate_test.go @@ -0,0 +1,28 @@ +package v2_test + +import ( + "testing" + + "github.com/stretchr/testify/require" + + keepertest "github.com/zeta-chain/zetacore/testutil/keeper" + v2 "github.com/zeta-chain/zetacore/x/authority/migrations/v2" + "github.com/zeta-chain/zetacore/x/authority/types" +) + +func TestMigrateStore(t *testing.T) { + t.Run("Set authorization list", func(t *testing.T) { + k, ctx := keepertest.AuthorityKeeper(t) + + list, found := k.GetAuthorizationList(ctx) + require.False(t, found) + require.Equal(t, types.AuthorizationList{}, list) + + err := v2.MigrateStore(ctx, *k) + require.NoError(t, err) + + list, found = k.GetAuthorizationList(ctx) + require.True(t, found) + require.Equal(t, types.DefaultAuthorizationsList(), list) + }) +} diff --git a/x/authority/module.go b/x/authority/module.go index f92da86e37..696c9c1edb 100644 --- a/x/authority/module.go +++ b/x/authority/module.go @@ -122,6 +122,10 @@ func (am AppModule) Name() string { func (am AppModule) RegisterServices(cfg module.Configurator) { types.RegisterMsgServer(cfg.MsgServer(), keeper.NewMsgServerImpl(am.keeper)) types.RegisterQueryServer(cfg.QueryServer(), am.keeper) + m := keeper.NewMigrator(am.keeper) + if err := cfg.RegisterMigration(types.ModuleName, 1, m.Migrate1to2); err != nil { + panic(err) + } } // RegisterInvariants registers the authority module's invariants. @@ -146,7 +150,7 @@ func (am AppModule) ExportGenesis(ctx sdk.Context, cdc codec.JSONCodec) json.Raw } // ConsensusVersion implements AppModule/ConsensusVersion. -func (AppModule) ConsensusVersion() uint64 { return 1 } +func (AppModule) ConsensusVersion() uint64 { return 2 } // BeginBlock executes all ABCI BeginBlock logic respective to the authority module. func (am AppModule) BeginBlock(_ sdk.Context, _ abci.RequestBeginBlock) {} diff --git a/x/authority/types/authorization_list_test.go b/x/authority/types/authorization_list_test.go index ee301c92d6..4a1cf701e8 100644 --- a/x/authority/types/authorization_list_test.go +++ b/x/authority/types/authorization_list_test.go @@ -209,6 +209,11 @@ func TestAuthorizationList_Validate(t *testing.T) { authorizations types.AuthorizationList expectedError error }{ + { + name: "validate default authorizations list", + authorizations: types.DefaultAuthorizationsList(), + expectedError: nil, + }, { name: "validate successfully", authorizations: types.AuthorizationList{Authorizations: []types.Authorization{