From b249faaafaf11baaa6b43a445aa0867b0097a36b Mon Sep 17 00:00:00 2001 From: Alex Gartner Date: Tue, 9 Jul 2024 12:27:05 -0700 Subject: [PATCH 1/5] fix(ci): prevent truncating the upgrade proposal with tee (#2453) --- contrib/localnet/scripts/start-upgrade-orchestrator.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/contrib/localnet/scripts/start-upgrade-orchestrator.sh b/contrib/localnet/scripts/start-upgrade-orchestrator.sh index 0bc93b7aec..1f8089bc3d 100755 --- a/contrib/localnet/scripts/start-upgrade-orchestrator.sh +++ b/contrib/localnet/scripts/start-upgrade-orchestrator.sh @@ -88,11 +88,11 @@ cat > upgrade_plan_info.json < Date: Tue, 9 Jul 2024 15:49:08 -0700 Subject: [PATCH 2/5] fix(ci): correct slack failure message conditional (#2457) --- .github/workflows/reusable-e2e.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/reusable-e2e.yml b/.github/workflows/reusable-e2e.yml index 2407598141..2bc75efcfb 100644 --- a/.github/workflows/reusable-e2e.yml +++ b/.github/workflows/reusable-e2e.yml @@ -101,7 +101,7 @@ jobs: path: /tmp/logs.txt - name: Notify Slack on Failure - if: failure() && (github.event_name == 'push' && github.ref == 'refs/heads/develop') || github.event_name == 'schedule' + if: failure() && ((github.event_name == 'push' && github.ref == 'refs/heads/develop') || github.event_name == 'schedule') uses: 8398a7/action-slack@v3 with: status: ${{ job.status }} From b602ded49f6e491fc4b82cb79307fbb404f365fd Mon Sep 17 00:00:00 2001 From: Alex Gartner Date: Wed, 10 Jul 2024 07:35:53 -0700 Subject: [PATCH 3/5] ci: optimize upgrade tests by downloading binaries (#2433) * ci: optimize upgrade tests by pulling image * invert conditional and parameterize --- Dockerfile-localnet | 28 +++++++++++++++++++--------- Makefile | 14 ++++++++++---- 2 files changed, 29 insertions(+), 13 deletions(-) diff --git a/Dockerfile-localnet b/Dockerfile-localnet index f9ff74d48a..24b98adf53 100644 --- a/Dockerfile-localnet +++ b/Dockerfile-localnet @@ -59,13 +59,9 @@ FROM base-runtime AS latest-runtime COPY --from=cosmovisor-build /go/bin/cosmovisor /usr/local/bin COPY --from=latest-build /go/bin/zetacored /go/bin/zetaclientd /go/bin/zetaclientd-supervisor /go/bin/zetae2e /usr/local/bin -# optional old version build. This old build is used as the genesis version in the upgrade tests. -# use --target latest-runtime to skip -# -# TODO: just download binaries from github release now that we're using glibc -# we can't do this right now since we do not have a v16 release candidate -# https://github.com/zeta-chain/node/issues/2179 -FROM base-build as old-build +# Optional old version build (from source). This old build is used as the genesis version in the upgrade tests. +# Use --target latest-runtime to skip. +FROM base-build as old-build-source ARG OLD_VERSION RUN git clone https://github.com/zeta-chain/node.git @@ -74,8 +70,22 @@ RUN cd node && git fetch RUN cd node && git checkout ${OLD_VERSION} RUN cd node && make install +FROM base-runtime AS old-runtime-source + +COPY --from=cosmovisor-build /go/bin/cosmovisor /usr/local/bin +COPY --from=old-build-source /go/bin/zetacored /go/bin/zetaclientd /usr/local/bin +COPY --from=latest-build /go/bin/zetaclientd-supervisor /usr/local/bin + +# Optional old version build (from binary). +# Use --target latest-runtime to skip. FROM base-runtime AS old-runtime +ARG OLD_VERSION +ARG BUILDARCH + COPY --from=cosmovisor-build /go/bin/cosmovisor /usr/local/bin -COPY --from=old-build /go/bin/zetacored /go/bin/zetaclientd /usr/local/bin -COPY --from=latest-build /go/bin/zetaclientd-supervisor /usr/local/bin \ No newline at end of file +COPY --from=latest-build /go/bin/zetaclientd-supervisor /usr/local/bin +RUN curl -Lo /usr/local/bin/zetacored ${OLD_VERSION}/zetacored-linux-${BUILDARCH} && \ + chmod 755 /usr/local/bin/zetacored && \ + curl -Lo /usr/local/bin/zetaclientd ${OLD_VERSION}/zetaclientd-linux-${BUILDARCH} && \ + chmod 755 /usr/local/bin/zetaclientd \ No newline at end of file diff --git a/Makefile b/Makefile index 177f064722..471c9af947 100644 --- a/Makefile +++ b/Makefile @@ -258,12 +258,18 @@ start-stress-test: zetanode ### Upgrade Tests ### ############################################################################### - +# build from source only if requested +ifdef UPGRADE_TEST_FROM_SOURCE zetanode-upgrade: zetanode - @echo "Building zetanode-upgrade" - $(DOCKER) build -t zetanode:old -f Dockerfile-localnet --target old-runtime --build-arg OLD_VERSION='release/v17' . - $(DOCKER) build -t orchestrator -f contrib/localnet/orchestrator/Dockerfile.fastbuild . + @echo "Building zetanode-upgrade from source" + $(DOCKER) build -t zetanode:old -f Dockerfile-localnet --target old-runtime-source --build-arg OLD_VERSION='release/v17' . +.PHONY: zetanode-upgrade +else +zetanode-upgrade: zetanode + @echo "Building zetanode-upgrade from binaries" + $(DOCKER) build -t zetanode:old -f Dockerfile-localnet --target old-runtime --build-arg OLD_VERSION='https://github.com/zeta-chain/ci-testing-node/releases/download/v17.0.1-internal' . .PHONY: zetanode-upgrade +endif start-upgrade-test: zetanode-upgrade @echo "--> Starting upgrade test" From 7e1dd537f2ae3d953550a04684c680ffdac5d2fb Mon Sep 17 00:00:00 2001 From: Grant Zukel <80433392+gzukel@users.noreply.github.com> Date: Wed, 10 Jul 2024 10:13:22 -0600 Subject: [PATCH 4/5] ci: add additional verification tweak concurrent users performance testing (#2425) * initial commit of tweaks for testing. * change user account * change user account * change user account * change user account * change user account * change user account * change user account * added option to specify external endpoint for testing and logic to support that with the scheduled nature of the job * added option to specify external endpoint for testing and logic to support that with the scheduled nature of the job * updated * updated --- .github/actions/performance-tests/art.yaml | 7 +- .../ci-nightly-performance-testing.yaml | 77 ++++++++++++++++++- changelog.md | 1 + 3 files changed, 79 insertions(+), 6 deletions(-) diff --git a/.github/actions/performance-tests/art.yaml b/.github/actions/performance-tests/art.yaml index 70d2ba1f85..35f79a788f 100644 --- a/.github/actions/performance-tests/art.yaml +++ b/.github/actions/performance-tests/art.yaml @@ -1,8 +1,8 @@ config: - target: "http://localhost:9545" + target: "-=http_request_protocol=-://-=endpoint_to_test=-" phases: - - duration: 60 - arrivalRate: 10 + - duration: 300 + arrivalRate: 400 defaults: headers: content-type: "application/json" @@ -19,7 +19,6 @@ config: - type: "html" filename: "artillery_report.html" logLevel: debug - scenarios: - name: web3_clientVersion flow: diff --git a/.github/workflows/ci-nightly-performance-testing.yaml b/.github/workflows/ci-nightly-performance-testing.yaml index a6a566f4aa..0c6280b135 100644 --- a/.github/workflows/ci-nightly-performance-testing.yaml +++ b/.github/workflows/ci-nightly-performance-testing.yaml @@ -2,6 +2,11 @@ name: "NIGHTLY:EVM:PERFORMANCE:TESTING" on: workflow_dispatch: + inputs: + endpoint: + description: 'The EVM endpoint you want to test.' + required: false + default: 'http://localhost:9545' schedule: - cron: '0 0 * * *' # Runs every day at midnight UTC @@ -9,6 +14,8 @@ jobs: nightly_evm_api_performance_test: name: "NIGHTLY:EVM:API:PERFORMANCE:TEST" runs-on: "buildjet-16vcpu-ubuntu-2204" + env: + endpoint_to_test: "http://localhost:9545" steps: - uses: actions/checkout@v4 @@ -25,16 +32,62 @@ jobs: run: | npm install -g artillery@latest + - name: "CONFIGURE:PERFORMANCE:TEMPLATE" + run: | + # Check if the inputs.endpoint starts with https:// or http:// + if [[ "${{inputs.endpoint}}" =~ ^https:// ]]; then + # Set the http_request_protocol to https if the URL starts with https:// + export http_request_protocol=https + elif [[ "${{inputs.endpoint}}" =~ ^http:// ]]; then + # Set the http_request_protocol to http if the URL starts with http:// + export http_request_protocol=http + else + # If the URL doesn't start with either http:// or https://, print an error and exit + echo "Please input a url with either https:// or http://" && exit 1 + fi + + # Check if inputs.endpoint is different from the default "http://localhost:9545" + if [ "${{inputs.endpoint}}" != "http://localhost:9545" ]; then + # Check if inputs.endpoint is not empty + if [ -n "${{inputs.endpoint}}" ]; then + # If inputs.endpoint isn't different from the default and is not empty, use inputs.endpoint + export endpoint_to_test=${{inputs.endpoint}} + # Remove the protocol (http:// or https://) from endpoint_to_test + endpoint_to_test=${endpoint_to_test#http://} + endpoint_to_test=${endpoint_to_test#https://} + else + # If inputs.endpoint is empty, strip the protocol from endpoint_to_test + endpoint_to_test=${endpoint_to_test#http://} + endpoint_to_test=${endpoint_to_test#https://} + fi + else + # If inputs.endpoint is "http://localhost:9545", strip the protocol from endpoint_to_test + endpoint_to_test=${endpoint_to_test#http://} + endpoint_to_test=${endpoint_to_test#https://} + fi + + # Replace -=endpoint_to_test=- placeholder in the art.yaml file with the endpoint_to_test value + sed -i "s/-=endpoint_to_test=-/${endpoint_to_test}/g" .github/actions/performance-tests/art.yaml + # Replace -=http_request_protocol=- placeholder in the art.yaml file with the http_request_protocol value + sed -i "s/-=http_request_protocol=-/${http_request_protocol}/g" .github/actions/performance-tests/art.yaml + - name: "EXECUTE:PERFORMANCE:TESTS" run: | - artillery run .github/actions/performance-tests/art.yaml --record --key ${{ secrets.ARTILLERY_KEY }} --output ./report.json + # Execute Artillery to run performance tests using the specified configuration file + artillery run .github/actions/performance-tests/art.yaml \ + --record \ + --key ${{ secrets.ARTILLERY_KEY }} \ + --output ./report.json + + - name: "CHECK:TEST:RESULTS" + run: | # Check Artillery exit status if [ $? -ne 0 ]; then echo "Artillery command failed to execute." exit 1 fi - # Parse the results.json file to check for failed vusers and http response codes + # Parse the results.json file to check for failed vusers and HTTP response codes failed_vusers=$(jq '.aggregate.counters["vusers.failed"] // 0' ./report.json) http_codes_200=$(jq '.aggregate.counters["http.codes.200"] // 0' ./report.json) http_responses=$(jq '.aggregate.counters["http.responses"] // 0' ./report.json) @@ -45,6 +98,26 @@ jobs: else echo "EVM Performance Testing Successful" fi + + # Read the JSON file and extract the required metrics + p99_values=$(jq -r '.aggregate.summaries["http.response_time"].p99, .aggregate.summaries["plugins.metrics-by-endpoint.response_time./"].p99, .aggregate.summaries["vusers.session_length"].p99' ./report.json) + p50_values=$(jq -r '.aggregate.summaries["http.response_time"].p50, .aggregate.summaries["plugins.metrics-by-endpoint.response_time./"].p50, .aggregate.summaries["vusers.session_length"].p50' ./report.json) + + # Check P99 values + for p99 in $p99_values; do + if (( $(echo "$p99 > 2000" | bc -l) )); then + echo "P99 value $p99 exceeds 2000ms" + exit 1 + fi + done + + # Check P50 values + for p50 in $p50_values; do + if (( $(echo "$p50 > 40" | bc -l) )); then + echo "P50 value $p50 exceeds 40ms" + exit 1 + fi + done - name: "GENERATE:REPORT" if: always() diff --git a/changelog.md b/changelog.md index 7b679fb678..621a03ef75 100644 --- a/changelog.md +++ b/changelog.md @@ -102,6 +102,7 @@ * [2335](https://github.com/zeta-chain/node/pull/2335) - ci: updated the artillery report to publish to artillery cloud * [2377](https://github.com/zeta-chain/node/pull/2377) - ci: adjusted sast-linters.yml to not scan itself, nor alert on removal of nosec. * [2400](https://github.com/zeta-chain/node/pull/2400) - ci: adjusted the performance test to pass or fail pipeline based on test results, alert slack, and launch network with state. Fixed connection issues as well. +* [2425](https://github.com/zeta-chain/node/pull/2425) - Added verification to performance testing pipeline to ensure p99 aren't above 2000ms and p50 aren't above 40ms, Tweaked the config to 400 users requests per second. 425 is the current max before it starts failing. ### Documentation From 9a527f96cf71651d5b8ca243474e48e6b73a2dbd Mon Sep 17 00:00:00 2001 From: Alex Gartner Date: Wed, 10 Jul 2024 11:05:25 -0700 Subject: [PATCH 5/5] fix: set pebbledb as default database (#2434) * fix: set pebbledb as default database * changelog entry --- changelog.md | 1 + cmd/zetacored/root.go | 2 ++ 2 files changed, 3 insertions(+) diff --git a/changelog.md b/changelog.md index 621a03ef75..887230bd31 100644 --- a/changelog.md +++ b/changelog.md @@ -89,6 +89,7 @@ * [2327](https://github.com/zeta-chain/node/pull/2327) - partially cherry picked the fix to Bitcoin outbound dust amount * [2362](https://github.com/zeta-chain/node/pull/2362) - set 1000 satoshis as minimum BTC amount that can be withdrawn from zEVM * [2382](https://github.com/zeta-chain/node/pull/2382) - add tx input and gas in rpc methods for synthetic eth txs +* [2434](https://github.com/zeta-chain/node/pull/2434) - the default database when running `zetacored init` is now pebbledb ### CI * [2388](https://github.com/zeta-chain/node/pull/2388) - added GitHub attestations of binaries produced in the release workflow. diff --git a/cmd/zetacored/root.go b/cmd/zetacored/root.go index 28654f885c..65a4e447c3 100644 --- a/cmd/zetacored/root.go +++ b/cmd/zetacored/root.go @@ -114,6 +114,8 @@ func initTmConfig() *tmcfg.Config { cfg.Mempool.Version = tmcfg.MempoolV1 } + cfg.DBBackend = "pebbledb" + return cfg }