diff --git a/.github/actions/performance-tests/art.yaml b/.github/actions/performance-tests/art.yaml index 71c1b3a15e..70d2ba1f85 100644 --- a/.github/actions/performance-tests/art.yaml +++ b/.github/actions/performance-tests/art.yaml @@ -1,7 +1,5 @@ -# artillery run art.yaml --output results.json -# artillery report results.json --output artillery_report.html config: - target: "http://127.0.0.1:8545" + target: "http://localhost:9545" phases: - duration: 60 arrivalRate: 10 @@ -13,10 +11,14 @@ config: plugins: metrics-by-endpoint: useOnlyRequestNames: true + ensure: + - type: "failure" + threshold: 1 summary: true reports: - type: "html" filename: "artillery_report.html" + logLevel: debug scenarios: - name: web3_clientVersion @@ -28,6 +30,15 @@ scenarios: jsonrpc: "2.0" method: "web3_clientVersion" params: [] + capture: + - json: "$" + as: "response" + ensure: + - statusCode: 200 + - statusCode: 201 + assert: + - subject: response.error + equals: null - name: web3_sha3 flow: @@ -39,6 +50,15 @@ scenarios: method: "web3_sha3" params: - "0x68656c6c6f20776f726c64" + capture: + - json: "$" + as: "response" + ensure: + - statusCode: 200 + - statusCode: 201 + assert: + - subject: response.error + equals: null - name: net_version flow: @@ -49,6 +69,15 @@ scenarios: jsonrpc: "2.0" method: "net_version" params: [] + capture: + - json: "$" + as: "response" + ensure: + - statusCode: 200 + - statusCode: 201 + assert: + - subject: response.error + equals: null - name: net_listening flow: @@ -59,6 +88,15 @@ scenarios: jsonrpc: "2.0" method: "net_listening" params: [] + capture: + - json: "$" + as: "response" + ensure: + - statusCode: 200 + - statusCode: 201 + assert: + - subject: response.error + equals: null - name: net_peerCount flow: @@ -69,6 +107,15 @@ scenarios: jsonrpc: "2.0" method: "net_peerCount" params: [] + capture: + - json: "$" + as: "response" + ensure: + - statusCode: 200 + - statusCode: 201 + assert: + - subject: response.error + equals: null - name: eth_chainId flow: @@ -79,6 +126,15 @@ scenarios: jsonrpc: "2.0" method: "eth_chainId" params: [] + capture: + - json: "$" + as: "response" + ensure: + - statusCode: 200 + - statusCode: 201 + assert: + - subject: response.error + equals: null - name: eth_getStorageAt flow: @@ -90,6 +146,15 @@ scenarios: method: "eth_getStorageAt" params: - "latest" + capture: + - json: "$" + as: "response" + ensure: + - statusCode: 200 + - statusCode: 201 + assert: + - subject: response.error + equals: null - name: eth_getTransactionByBlockHashAndIndex flow: @@ -101,6 +166,15 @@ scenarios: method: "eth_getTransactionByBlockHashAndIndex" params: - "0x0" + capture: + - json: "$" + as: "response" + ensure: + - statusCode: 200 + - statusCode: 201 + assert: + - subject: response.error + equals: null - name: eth_getTransactionByBlockNumberAndIndex flow: @@ -113,6 +187,15 @@ scenarios: params: - "latest" - "0x0" + capture: + - json: "$" + as: "response" + ensure: + - statusCode: 200 + - statusCode: 201 + assert: + - subject: response.error + equals: null - name: eth_getUncleByBlockHashAndIndex flow: @@ -124,6 +207,15 @@ scenarios: method: "eth_getUncleByBlockHashAndIndex" params: - "0x0" + capture: + - json: "$" + as: "response" + ensure: + - statusCode: 200 + - statusCode: 201 + assert: + - subject: response.error + equals: null - name: eth_getUncleByBlockNumberAndIndex flow: @@ -136,6 +228,15 @@ scenarios: params: - "latest" - "0x0" + capture: + - json: "$" + as: "response" + ensure: + - statusCode: 200 + - statusCode: 201 + assert: + - subject: response.error + equals: null - name: eth_newFilter flow: @@ -152,6 +253,15 @@ scenarios: address: "0x0", topics: [] } + capture: + - json: "$" + as: "response" + ensure: + - statusCode: 200 + - statusCode: 201 + assert: + - subject: response.error + equals: null - name: eth_getFilterChanges flow: @@ -163,6 +273,15 @@ scenarios: method: "eth_getFilterChanges" params: - "0x0" + capture: + - json: "$" + as: "response" + ensure: + - statusCode: 200 + - statusCode: 201 + assert: + - subject: response.error + equals: null - name: eth_getFilterLogs flow: @@ -174,6 +293,15 @@ scenarios: method: "eth_getFilterLogs" params: - "0x0" + capture: + - json: "$" + as: "response" + ensure: + - statusCode: 200 + - statusCode: 201 + assert: + - subject: response.error + equals: null - name: eth_blockNumber flow: @@ -184,6 +312,15 @@ scenarios: jsonrpc: "2.0" method: "eth_blockNumber" params: [] + capture: + - json: "$" + as: "response" + ensure: + - statusCode: 200 + - statusCode: 201 + assert: + - subject: response.error + equals: null - name: eth_getBlockByHash flow: @@ -196,6 +333,15 @@ scenarios: params: - "0x0" - true + capture: + - json: "$" + as: "response" + ensure: + - statusCode: 200 + - statusCode: 201 + assert: + - subject: response.error + equals: null - name: eth_getBlockByNumber flow: @@ -208,6 +354,15 @@ scenarios: params: - "latest" - true + capture: + - json: "$" + as: "response" + ensure: + - statusCode: 200 + - statusCode: 201 + assert: + - subject: response.error + equals: null - name: eth_getBlockTransactionCountByHash flow: @@ -219,6 +374,15 @@ scenarios: method: "eth_getBlockTransactionCountByHash" params: - "0x0" + capture: + - json: "$" + as: "response" + ensure: + - statusCode: 200 + - statusCode: 201 + assert: + - subject: response.error + equals: null - name: eth_getBlockTransactionCountByNumber flow: @@ -230,6 +394,15 @@ scenarios: method: "eth_getBlockTransactionCountByNumber" params: - "latest" + capture: + - json: "$" + as: "response" + ensure: + - statusCode: 200 + - statusCode: 201 + assert: + - subject: response.error + equals: null - name: eth_getCode flow: @@ -242,6 +415,15 @@ scenarios: params: - "0x0" - "latest" + capture: + - json: "$" + as: "response" + ensure: + - statusCode: 200 + - statusCode: 201 + assert: + - subject: response.error + equals: null - name: eth_getTransactionByHash flow: @@ -253,6 +435,15 @@ scenarios: method: "eth_getTransactionByHash" params: - "0x0" + capture: + - json: "$" + as: "response" + ensure: + - statusCode: 200 + - statusCode: 201 + assert: + - subject: response.error + equals: null - name: eth_getTransactionReceipt flow: @@ -264,6 +455,15 @@ scenarios: method: "eth_getTransactionReceipt" params: - "0x0" + capture: + - json: "$" + as: "response" + ensure: + - statusCode: 200 + - statusCode: 201 + assert: + - subject: response.error + equals: null - name: eth_getLogs flow: @@ -278,6 +478,15 @@ scenarios: toBlock: "latest" address: "0x0" topics: ["0x0"] + capture: + - json: "$" + as: "response" + ensure: + - statusCode: 200 + - statusCode: 201 + assert: + - subject: response.error + equals: null - name: eth_getBalance flow: @@ -290,6 +499,15 @@ scenarios: params: - "0x0" - "latest" + capture: + - json: "$" + as: "response" + ensure: + - statusCode: 200 + - statusCode: 201 + assert: + - subject: response.error + equals: null - name: eth_estimateGas flow: @@ -301,3 +519,12 @@ scenarios: method: "eth_estimateGas" params: - from: "0x0" + capture: + - json: "$" + as: "response" + ensure: + - statusCode: 200 + - statusCode: 201 + assert: + - subject: response.error + equals: null diff --git a/.github/workflows/ci-nightly-performance-testing.yaml b/.github/workflows/ci-nightly-performance-testing.yaml index 30116f743f..a6a566f4aa 100644 --- a/.github/workflows/ci-nightly-performance-testing.yaml +++ b/.github/workflows/ci-nightly-performance-testing.yaml @@ -12,23 +12,57 @@ jobs: steps: - uses: actions/checkout@v4 - - name: "START:LOCAL:NET" + - name: "INSTALL:NODEJS" + uses: actions/setup-node@v4 + with: + node-version: 16 + + - name: "START:LOCAL:NET:WITH:STATE" run: | - make start-e2e-test + make start-e2e-import-mainnet-test - - name: "EXECUTE:LOADTESTS" - uses: artilleryio/action-cli@v1 - with: - command: run .github/actions/performance-tests/art.yaml --record --key ${{ secrets.ARTILLERY_KEY }} --output ./report.json + - name: "INSTALL:ARTILLERY" + run: | + npm install -g artillery@latest + + - name: "EXECUTE:PERFORMANCE:TESTS" + run: | + artillery run .github/actions/performance-tests/art.yaml --record --key ${{ secrets.ARTILLERY_KEY }} --output ./report.json + # Check Artillery exit status + if [ $? -ne 0 ]; then + echo "Artillery command failed to execute." + exit 1 + fi + + # Parse the results.json file to check for failed vusers and http response codes + failed_vusers=$(jq '.aggregate.counters["vusers.failed"] // 0' ./report.json) + http_codes_200=$(jq '.aggregate.counters["http.codes.200"] // 0' ./report.json) + http_responses=$(jq '.aggregate.counters["http.responses"] // 0' ./report.json) + + if [ "$failed_vusers" -gt 1 ] || [ "$http_codes_200" -ne "$http_responses" ]; then + echo "Performance Testing had Failed Tests." + exit 1 + else + echo "EVM Performance Testing Successful" + fi - name: "GENERATE:REPORT" - uses: artilleryio/action-cli@v1 - with: - command: report report.json --output artillery_report.html + if: always() + run: | + artillery report report.json --output artillery_report.html - name: "UPLOAD:REPORT" uses: actions/upload-artifact@v3 - if: success() + if: always() with: name: artillery-report - path: ./artillery_report.html \ No newline at end of file + path: ./artillery_report.html + + - name: "NOTIFY:SLACK:FAILURE" + if: failure() && github.event_name == 'schedule' + uses: 8398a7/action-slack@v3 + with: + status: ${{ job.status }} + fields: repo,message,commit,author,action,eventName,ref,workflow,job,took + env: + SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_CI_ALERTS }} \ No newline at end of file diff --git a/changelog.md b/changelog.md index 4e7cd838a0..1506dee6ee 100644 --- a/changelog.md +++ b/changelog.md @@ -99,6 +99,7 @@ * [2192](https://github.com/zeta-chain/node/pull/2192) - Added release status checker and updater pipeline that will update release statuses when they go live on network * [2335](https://github.com/zeta-chain/node/pull/2335) - ci: updated the artillery report to publish to artillery cloud * [2377](https://github.com/zeta-chain/node/pull/2377) - ci: adjusted sast-linters.yml to not scan itself, nor alert on removal of nosec. +* [2400](https://github.com/zeta-chain/node/pull/2400) - ci: adjusted the performance test to pass or fail pipeline based on test results, alert slack, and launch network with state. Fixed connection issues as well. ### Documentation