ci-nightly-performance-testing.yaml

name: "NIGHTLY:EVM:PERFORMANCE:TESTING"

on:
  workflow_dispatch:
    inputs:
      endpoint:
        description: 'The EVM endpoint you want to test.'
        required: false
        default: 'http://localhost:9545'
  schedule:
    # run at 6AM UTC Daily
    # 6AM UTC -> 11PM PT
    - cron: "0 6 * * *"

jobs:
  nightly_evm_api_performance_test:
    name: "NIGHTLY:EVM:API:PERFORMANCE:TEST"
    runs-on: "buildjet-16vcpu-ubuntu-2204"
    env:
      endpoint_to_test: "http://localhost:9545"
    steps:
      - uses: actions/checkout@v4

      - name: "INSTALL:NODEJS"
        uses: actions/setup-node@v4
        with:
          node-version: 20

      - name: "START:LOCAL:NET:WITH:STATE"
        run: |
          make start-e2e-import-mainnet-test

      - name: "INSTALL:ARTILLERY"
        run: |
          npm install -g artillery@latest

      - name: "CONFIGURE:PERFORMANCE:TEMPLATE"
        env:
          CLOUDFLARE_UUID: ${{ secrets.CLOUDFLARE_UUID }}
          ENDPOINT: ${{ inputs.endpoint || 'http://localhost:9545' }}
          GITHUB_ACTION: ${{ github.action }}
        run: |
          # Replace -=endpoint_to_test=- placeholder in the art.yaml file with the endpoint_to_test value
          sed -i "s|-=endpoint_to_test=-|${ENDPOINT}|g" .github/actions/performance-tests/art.yaml
          # Replace -=CLOUDFLARE_UUID=- placeholder in the art.yaml file with the correct value
          sed -i "s|-=CLOUDFLARE_UUID=-|${CLOUDFLARE_UUID}|g" .github/actions/performance-tests/art.yaml
          # Replace -=GITHUB_ACTION=- placeholder in the art.yaml file with the correct value
          sed -i "s|-=GITHUB_ACTION=-|${GITHUB_ACTION}|g" .github/actions/performance-tests/art.yaml
          cat .github/actions/performance-tests/art.yaml

      - name: "EXECUTE:PERFORMANCE:TESTS"
        run: |
          # Execute Artillery to run performance tests using the specified configuration file
          artillery run .github/actions/performance-tests/art.yaml \
          --record \
          --key ${{ secrets.ARTILLERY_KEY }} \
          --output ./report.json

      - name: "CHECK:TEST:RESULTS"
        run: |
          # Check Artillery exit status
          if [ $? -ne 0 ]; then
            echo "Artillery command failed to execute."
            exit 1
          fi
          
          # Parse the results.json file to check for failed vusers and HTTP response codes
          failed_vusers=$(jq '.aggregate.counters["vusers.failed"] // 0' ./report.json)
          http_codes_200=$(jq '.aggregate.counters["http.codes.200"] // 0' ./report.json)
          http_responses=$(jq '.aggregate.counters["http.responses"] // 0' ./report.json)
          
          if [ "$failed_vusers" -gt 1 ] || [ "$http_codes_200" -ne "$http_responses" ]; then
            echo "Performance Testing had Failed Tests."
            exit 1
          else
            echo "EVM Performance Testing Successful"
          fi
          
          # Read the JSON file and extract the required metrics
          p99_values=$(jq -r '.aggregate.summaries["http.response_time"].p99, .aggregate.summaries["plugins.metrics-by-endpoint.response_time./"].p99, .aggregate.summaries["vusers.session_length"].p99' ./report.json)
          p50_values=$(jq -r '.aggregate.summaries["http.response_time"].p50, .aggregate.summaries["plugins.metrics-by-endpoint.response_time./"].p50, .aggregate.summaries["vusers.session_length"].p50' ./report.json)
          
          # Check P99 values
          for p99 in $p99_values; do
            if (( $(echo "$p99 > 2000" | bc -l) )); then
              echo "P99 value $p99 exceeds 2000ms"
              exit 1
            fi
          done
          
          # Check P50 values
          for p50 in $p50_values; do
            if (( $(echo "$p50 > 40" | bc -l) )); then
              echo "P50 value $p50 exceeds 40ms"
              exit 1
            fi
          done

      - name: "GENERATE:REPORT"
        if: always()
        run: |
          artillery report report.json --output artillery_report.html

      - name: "UPLOAD:REPORT"
        uses: actions/upload-artifact@v4
        if: always()
        with:
          name: artillery-report
          path: ./artillery_report.html

      - name: "NOTIFY:SLACK:FAILURE"
        if: failure() && github.event_name == 'schedule'
        uses: 8398a7/action-slack@v3
        with:
          status: ${{ job.status }}
          fields: repo,message,commit,author,action,eventName,ref,workflow,job,took
          job_name: "NIGHTLY:EVM:API:PERFORMANCE:TEST"
        env:
          SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_CI_ALERTS }}