NIGHTLY:EVM:PERFORMANCE:TESTING #104
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: "NIGHTLY:EVM:PERFORMANCE:TESTING" | |
| on: | |
| workflow_dispatch: | |
| inputs: | |
| endpoint: | |
| description: 'The EVM endpoint you want to test.' | |
| required: false | |
| default: 'http://localhost:9545' | |
| schedule: | |
| - cron: '0 0 * * *' # Runs every day at midnight UTC | |
| jobs: | |
| nightly_evm_api_performance_test: | |
| name: "NIGHTLY:EVM:API:PERFORMANCE:TEST" | |
| runs-on: "buildjet-16vcpu-ubuntu-2204" | |
| env: | |
| endpoint_to_test: "http://localhost:9545" | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: "INSTALL:NODEJS" | |
| uses: actions/setup-node@v4 | |
| with: | |
| node-version: 16 | |
| - name: "START:LOCAL:NET:WITH:STATE" | |
| run: | | |
| make start-e2e-import-mainnet-test | |
| - name: "INSTALL:ARTILLERY" | |
| run: | | |
| npm install -g artillery@latest | |
| - name: "CONFIGURE:PERFORMANCE:TEMPLATE" | |
| env: | |
| CLOUDFLARE_UUID: ${{ secrets.CLOUDFLARE_UUID }} | |
| run: | | |
| # Check if the inputs.endpoint starts with https:// or http:// | |
| if [[ "${{inputs.endpoint}}" =~ ^https:// ]]; then | |
| # Set the http_request_protocol to https if the URL starts with https:// | |
| export http_request_protocol=https | |
| elif [[ "${{inputs.endpoint}}" =~ ^http:// ]]; then | |
| # Set the http_request_protocol to http if the URL starts with http:// | |
| export http_request_protocol=http | |
| else | |
| # If the URL doesn't start with either http:// or https://, print an error and exit | |
| echo "Please input a url with either https:// or http://" && exit 1 | |
| fi | |
| # Check if inputs.endpoint is different from the default "http://localhost:9545" | |
| if [ "${{inputs.endpoint}}" != "http://localhost:9545" ]; then | |
| # Check if inputs.endpoint is not empty | |
| if [ -n "${{inputs.endpoint}}" ]; then | |
| # If inputs.endpoint isn't different from the default and is not empty, use inputs.endpoint | |
| export endpoint_to_test=${{inputs.endpoint}} | |
| # Remove the protocol (http:// or https://) from endpoint_to_test | |
| endpoint_to_test=${endpoint_to_test#http://} | |
| endpoint_to_test=${endpoint_to_test#https://} | |
| else | |
| # If inputs.endpoint is empty, strip the protocol from endpoint_to_test | |
| endpoint_to_test=${endpoint_to_test#http://} | |
| endpoint_to_test=${endpoint_to_test#https://} | |
| fi | |
| else | |
| # If inputs.endpoint is "http://localhost:9545", strip the protocol from endpoint_to_test | |
| endpoint_to_test=${endpoint_to_test#http://} | |
| endpoint_to_test=${endpoint_to_test#https://} | |
| fi | |
| # Replace -=endpoint_to_test=- placeholder in the art.yaml file with the endpoint_to_test value | |
| sed -i "s/-=endpoint_to_test=-/${endpoint_to_test}/g" .github/actions/performance-tests/art.yaml | |
| # Replace -=http_request_protocol=- placeholder in the art.yaml file with the http_request_protocol value | |
| sed -i "s/-=http_request_protocol=-/${http_request_protocol}/g" .github/actions/performance-tests/art.yaml | |
| - name: "EXECUTE:PERFORMANCE:TESTS" | |
| env: | |
| CLOUDFLARE_UUID: ${{ secrets.CLOUDFLARE_UUID }} | |
| run: | | |
| # Execute Artillery to run performance tests using the specified configuration file | |
| artillery run .github/actions/performance-tests/art.yaml \ | |
| --record \ | |
| --key ${{ secrets.ARTILLERY_KEY }} \ | |
| --output ./report.json | |
| - name: "CHECK:TEST:RESULTS" | |
| run: | | |
| # Check Artillery exit status | |
| if [ $? -ne 0 ]; then | |
| echo "Artillery command failed to execute." | |
| exit 1 | |
| fi | |
| # Parse the results.json file to check for failed vusers and HTTP response codes | |
| failed_vusers=$(jq '.aggregate.counters["vusers.failed"] // 0' ./report.json) | |
| http_codes_200=$(jq '.aggregate.counters["http.codes.200"] // 0' ./report.json) | |
| http_responses=$(jq '.aggregate.counters["http.responses"] // 0' ./report.json) | |
| if [ "$failed_vusers" -gt 1 ] || [ "$http_codes_200" -ne "$http_responses" ]; then | |
| echo "Performance Testing had Failed Tests." | |
| exit 1 | |
| else | |
| echo "EVM Performance Testing Successful" | |
| fi | |
| # Read the JSON file and extract the required metrics | |
| p99_values=$(jq -r '.aggregate.summaries["http.response_time"].p99, .aggregate.summaries["plugins.metrics-by-endpoint.response_time./"].p99, .aggregate.summaries["vusers.session_length"].p99' ./report.json) | |
| p50_values=$(jq -r '.aggregate.summaries["http.response_time"].p50, .aggregate.summaries["plugins.metrics-by-endpoint.response_time./"].p50, .aggregate.summaries["vusers.session_length"].p50' ./report.json) | |
| # Check P99 values | |
| for p99 in $p99_values; do | |
| if (( $(echo "$p99 > 2000" | bc -l) )); then | |
| echo "P99 value $p99 exceeds 2000ms" | |
| exit 1 | |
| fi | |
| done | |
| # Check P50 values | |
| for p50 in $p50_values; do | |
| if (( $(echo "$p50 > 40" | bc -l) )); then | |
| echo "P50 value $p50 exceeds 40ms" | |
| exit 1 | |
| fi | |
| done | |
| - name: "GENERATE:REPORT" | |
| if: always() | |
| run: | | |
| artillery report report.json --output artillery_report.html | |
| - name: "UPLOAD:REPORT" | |
| uses: actions/upload-artifact@v3 | |
| if: always() | |
| with: | |
| name: artillery-report | |
| path: ./artillery_report.html | |
| - name: "NOTIFY:SLACK:FAILURE" | |
| if: failure() && github.event_name == 'schedule' | |
| uses: 8398a7/action-slack@v3 | |
| with: | |
| status: ${{ job.status }} | |
| fields: repo,message,commit,author,action,eventName,ref,workflow,job,took | |
| env: | |
| SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_CI_ALERTS }} |