hil.yml

name: Run tests

on:
  pull_request:
    types: [opened, reopened, synchronize, labeled]

  schedule:
    - cron: '0 2 * * *'

env:
  MAX_CHUNKS: 15
  WOKWI_TIMEOUT: 120000 # Milliseconds
  WOKWI_CLI_TOKEN: ${{ secrets.WOKWI_CLI_TOKEN }}

concurrency:
  group: hil-${{github.event.pull_request.number || github.ref}}
  cancel-in-progress: true

jobs:
  gen_chunks:
    if: |
      github.event_name == 'pull_request' ||
      (github.event_name == 'schedule' && github.repository == 'espressif/arduino-esp32')
    name: Generate Chunks matrix
    runs-on: ubuntu-latest
    outputs:
      chunks: ${{ steps.gen-chunks.outputs.chunks }}
      test_folder: ${{ steps.gen-chunks.outputs.test_folder }}
      test_type: ${{ steps.gen-chunks.outputs.test_type }}
    steps:
      - name: Checkout Repository
        uses: actions/checkout@v4

      - name: Generate Chunks matrix
        id: gen-chunks
        run: |
          set +e
          if [ "${{contains(github.event.pull_request.labels.*.name, 'hil_test')}}" == "true" ] && \
             [ "${{contains(github.event.pull_request.labels.*.name, 'perf_test')}}" == "false" ]; then
            test_folder="tests/validation"
            test_type="validation"
          elif [ "${{contains(github.event.pull_request.labels.*.name, 'hil_test')}}" == "false" ] && \
               [ "${{contains(github.event.pull_request.labels.*.name, 'perf_test')}}" == "true" ]; then
            test_folder="tests/performance"
            test_type="performance"
          else
            test_folder="tests"
            test_type="all"
          fi
          .github/scripts/sketch_utils.sh count $test_folder
          sketches=$?
          if [[ $sketches -ge ${{env.MAX_CHUNKS}} ]]; then
            $sketches=${{env.MAX_CHUNKS}}
          fi
          set -e
          rm sketches.txt
          CHUNKS=$(jq -c -n '$ARGS.positional' --args `seq 0 1 $((sketches - 1))`)
          echo "chunks=${CHUNKS}" >> $GITHUB_OUTPUT
          echo "test_folder=${test_folder}" >> $GITHUB_OUTPUT
          echo "test_type=${test_type}" >> $GITHUB_OUTPUT

      - name: Upload Event file
        uses: actions/upload-artifact@v4
        with:
          name: event_file
          path: ${{github.event_path}}

  build:
    needs: gen_chunks
    name: ${{matrix.chip}}-Build#${{matrix.chunks}}
    runs-on: ubuntu-latest
    strategy:
      matrix:
        chip: ['esp32', 'esp32s2', 'esp32s3', 'esp32c3', 'esp32c6', 'esp32h2']
        chunks: ${{fromJson(needs.gen_chunks.outputs.chunks)}}
    steps:
      - name: Checkout Repository
        uses: actions/checkout@v4

      - name: Build sketches
        run: |
          bash .github/scripts/tests_build.sh -c -type ${{ needs.gen_chunks.outputs.test_type }} -t ${{matrix.chip}} -i ${{matrix.chunks}} -m ${{env.MAX_CHUNKS}}

      - name: Upload ${{matrix.chip}}-${{matrix.chunks}} artifacts
        uses: actions/upload-artifact@v4
        with:
          name: ${{matrix.chip}}-${{matrix.chunks}}.artifacts
          if-no-files-found: error
          path: |
            ~/.build_skipped
            ~/.arduino/tests/**/build*.tmp/*.bin
            ~/.arduino/tests/**/build*.tmp/*.elf
            ~/.arduino/tests/**/build*.tmp/*.json

  qemu-test:
    needs: [gen_chunks, build]
    name: ${{matrix.chip}}-QEMU_Test#${{matrix.chunks}}
    if: ${{ false }}
    strategy:
      fail-fast: false
      matrix:
        chip: ['esp32', 'esp32c3'] # Currently only ESP32 and ESP32-C3 are supported by QEMU
        chunks: ${{fromJson(needs.gen_chunks.outputs.chunks)}}
    runs-on: ubuntu-latest
    env:
      QEMU_INSTALL_PATH: "$HOME"
    steps:
      - name: Checkout repository
        uses: actions/checkout@v4

      - name: Get QEMU version
        uses: pozetroninc/github-action-get-latest-release@v0.7.0
        id: get-qemu-version
        with:
          token: ${{secrets.GITHUB_TOKEN}}
          owner: espressif
          repo: qemu
          excludes: prerelease, draft

      - name: Cache tools
        id: cache-linux
        uses: actions/cache@v4
        with:
          path: |
            ~/qemu
            ~/.cache/pip
          key: qemu-${{ steps.get-qemu-version.outputs.release }}-${{ hashFiles('.github/workflows/hil.yml') }}

      - name: Install dependencies
        run: |
          pip install -U pip
          pip install -r tests/requirements.txt --extra-index-url https://dl.espressif.com/pypi
          sudo apt update && sudo apt install libpixman-1-0 libnuma1 libglib2.0-0 libslirp0 libsdl2-2.0-0

      - name: Download QEMU
        if: steps.cache-linux.outputs.cache-hit != 'true'
        run: |
          cd ${{ env.QEMU_INSTALL_PATH }}
          underscore_release=$(echo ${{ steps.get-qemu-version.outputs.release }} | sed 's/\-/_/g')
          curl -L https://github.com/espressif/qemu/releases/download/${{ steps.get-qemu-version.outputs.release }}/qemu-riscv32-softmmu-${underscore_release}-x86_64-linux-gnu.tar.xz > qemu-riscv32.tar.xz
          curl -L https://github.com/espressif/qemu/releases/download/${{ steps.get-qemu-version.outputs.release }}/qemu-xtensa-softmmu-${underscore_release}-x86_64-linux-gnu.tar.xz > qemu-xtensa.tar.xz
          tar -xf qemu-riscv32.tar.xz
          tar -xf qemu-xtensa.tar.xz
          rm qemu-*
          echo "QEMU_PATH=${{ env.QEMU_INSTALL_PATH }}/qemu" >> $GITHUB_ENV

      - name: Download ${{matrix.chip}}-${{matrix.chunks}} artifacts
        uses: actions/download-artifact@v4
        with:
          name: ${{matrix.chip}}-${{matrix.chunks}}.artifacts
          path: ~/

      - name: Run Tests
        run: QEMU_PATH="${{env.QEMU_PATH}}" bash .github/scripts/tests_run.sh -c -t ${{matrix.chip}} -i ${{matrix.chunks}} -m ${{env.MAX_CHUNKS}} -q

      - name: Check if tests were skipped
        id: check-test-skipped
        run: |
          if [ $(find "tests" -name ".test_skipped") ]; then
            echo "skipped=true" >> $GITHUB_OUTPUT
          else
            echo "skipped=false" >> $GITHUB_OUTPUT
          fi

      - name: Upload test result artifacts
        uses: actions/upload-artifact@v4
        if: ${{ always() && steps.check-test-skipped.outputs.skipped == 'false' }}
        with:
          name: qemu_results-${{matrix.chip}}-${{matrix.chunks}}
          path: tests/*/*.xml

  wokwi-test:
    needs: [gen_chunks, build]
    if: github.event_name == 'schedule'
    name: ${{matrix.chip}}-Wokwi_Test#${{matrix.chunks}}
    strategy:
      fail-fast: false
      matrix:
        chip: ['esp32', 'esp32s2', 'esp32s3', 'esp32c3', 'esp32c6', 'esp32h2']
        chunks: ${{fromJson(needs.gen_chunks.outputs.chunks)}}
    runs-on: ubuntu-latest
    steps:
      - name: Checkout repository
        uses: actions/checkout@v4

      - name: Download ${{matrix.chip}}-${{matrix.chunks}} artifacts
        uses: actions/download-artifact@v4
        with:
          name: ${{matrix.chip}}-${{matrix.chunks}}.artifacts
          path: ~/

      - name: Install Wokwi CLI
        run: curl -L https://wokwi.com/ci/install.sh | sh

      - name: Install dependencies
        run: |
          pip install -U pip
          pip install -r tests/requirements.txt --extra-index-url https://dl.espressif.com/pypi
          sudo apt update && sudo apt install -y -qq jq

      - name: Run Tests
        env:
          WOKWI_CLI_TOKEN: ${{ secrets.WOKWI_CLI_TOKEN }}
        run: |
          bash .github/scripts/tests_run.sh -c -t ${{matrix.chip}} -i ${{matrix.chunks}} -m ${{env.MAX_CHUNKS}} -w ${{env.WOKWI_TIMEOUT}}

      - name: Check if tests were skipped
        id: check-test-skipped
        run: |
          if [ $(find "tests" -name ".test_skipped") ]; then
            echo "skipped=true" >> $GITHUB_OUTPUT
          else
            echo "skipped=false" >> $GITHUB_OUTPUT
          fi

      - name: Upload test result artifacts
        uses: actions/upload-artifact@v4
        if: ${{ always() && steps.check-test-skipped.outputs.skipped == 'false' }}
        with:
          name: wokwi_results-${{matrix.chip}}-${{matrix.chunks}}
          path: tests/**/*.xml

  hardware-test:
    needs: [gen_chunks, build]
    name: ${{matrix.chip}}-Hardware_Test#${{matrix.chunks}}
    if: |
      contains(github.event.pull_request.labels.*.name, 'hil_test') || github.event_name == 'schedule'
    strategy:
      fail-fast: false
      matrix:
        chip: ['esp32', 'esp32s2', 'esp32s3', 'esp32c3', 'esp32c6', 'esp32h2']
        chunks: ${{fromJson(needs.gen_chunks.outputs.chunks)}}
    runs-on: [arduino, "${{matrix.chip}}"]
    container:
      image: python:3.10.1-bullseye
      options: --privileged

    steps:
      - name: Checkout repository
        uses: actions/checkout@v4

      - name: Download ${{matrix.chip}}-${{matrix.chunks}} artifacts
        uses: actions/download-artifact@v4
        with:
          name: ${{matrix.chip}}-${{matrix.chunks}}.artifacts
          path: ~/

      - name: Install dependencies
        run: |
          pip install -U pip
          pip install -r tests/requirements.txt --extra-index-url https://dl.espressif.com/pypi
          apt update && apt install -y -qq jq

      - name: Run Tests
        run: |
          bash .github/scripts/tests_run.sh -c -type ${{ needs.gen_chunks.outputs.test_type }} -t ${{matrix.chip}} -i ${{matrix.chunks}} -m ${{env.MAX_CHUNKS}} -e

      - name: Check if tests were skipped
        id: check-test-skipped
        run: |
          if [ $(find "tests" -name ".test_skipped") ]; then
            echo "skipped=true" >> $GITHUB_OUTPUT
          else
            echo "skipped=false" >> $GITHUB_OUTPUT
          fi

      - name: Upload test result artifacts
        uses: actions/upload-artifact@v4
        if: ${{ always() && steps.check-test-skipped.outputs.skipped == 'false' }}
        with:
          name: hw_results-${{matrix.chip}}-${{matrix.chunks}}
          if-no-files-found: error
          path: |
            tests/**/*.xml
            tests/**/result_*.json