main.yml

on:
  # Every pull request
  pull_request:
  workflow_dispatch:
    # Allows manually triggering release
    inputs:
      release_version:
        description: "Version/branch to benchmark. See README.md for syntax. (defaults to main branch)"
        required: false
        default: ""
      filter_experiments:
        description: "Limit the benchmark runs to bench suites matching the comma separated names -- leave blank to run all defined experiments"
        required: false
        default: ""
      encoding_comparison_max_length:
        description: "Set checking length for encoding comparison benchmarks (defaults to 8)"
        required: false
        default: ""
  # Disable scheduled runs, since the benchmarks are broken
  # schedule:
    # * is a special character in YAML so you have to quote this string
    # Automatically run the benchmarks on main every Saturday
    # See https://crontab.guru/#0_0_*_*_6
    # - cron: "0 0 * * 6"

name: benchmarks

env:
  VERSION: ${{ github.event.inputs.release_version || '#main' }}
  FILTER_EXPERIMENTS: ${{ github.event.inputs.filter_experiments }}
  ENCODING_COMPARISON_MAX_LENGTH: ${{ github.event.inputs.encoding_comparison_max_length }}
  # GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

jobs:
  run-benchmarks:
    runs-on: ubuntu-20.04 # BenchExec requires cgroupsv1 support, see #112
    steps:
      - uses: actions/checkout@v2
        with:
          # Required to enable pushing report changes back to the branch
          # See https://stackoverflow.com/a/58393457/1187277
          ref: ${{ github.head_ref }}
      - name: Set up JDK 1.8
        uses: actions/setup-java@v1
        with:
          java-version: 1.8
      - name: Retrieve SBT Cache
        uses: coursier/cache-action@v6
      - name: Install benchexec
        run: |
          sudo add-apt-repository --yes ppa:sosy-lab/benchmarking
          sudo apt install --yes benchexec
          # Workaround for inability to set persistant user groups
          # across shells
          sudo chmod o+wt /sys/fs/cgroup/blkio/system.slice/benchexec-cgroup.service /sys/fs/cgroup/cpu,cpuacct/system.slice/benchexec-cgroup.service /sys/fs/cgroup/cpuset/system.slice/benchexec-cgroup.service /sys/fs/cgroup/freezer/system.slice/benchexec-cgroup.service /sys/fs/cgroup/memory/system.slice/benchexec-cgroup.service
      - name: Check benchexec configuration
        run: python3 -m benchexec.check_cgroups
      - name: Run benchmarks
        env:
          # Cannot run benchexec in a container inside of github runners
          BENCH_CONTAINER: false
          # Set to `true` to enable debug output for benchexec
          BENCH_DEBUG: true
        run: |
          # Ensure we have the needed dirs in PATH and PYTHONPATH
          source ./.envrc
          sbt 'set apalacheVersion := "${{ env.VERSION }}"; set ThisBuild/benchmarksFilterExperiments := Set(${{ env.FILTER_EXPERIMENTS }}); benchmarksReport'
      - name: Report any errors
        # TODO
        # ONLY RUN IF ON TRUNK
        # if: github.ref == 'refs/heads/trunk'
        run: |
          err_files=$(find -name 'ERRORS.*.json')
          if [[ -n "$err_files" ]]
          then
            hub issue create \
              --message="Errors detected in benchmark run" \
              --message='```' \
              --message="""$(cat $err_files)""" \
              --message='```' \
              --assign=shonfeder
          fi
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
      - name: Generate site and report
        run: sbt site/benchmarksLongitudinalUpdate site/benchmarksIndexUpdate makeSite
      - name: Commit updated results
        # ONLY RUN IF ON TRUNK
        if: github.ref == 'refs/heads/trunk'
        run: |
          git config --global user.name 'apalache-bench-ci'
          git config --global user.email 'apalache-bench-ci@users.noreply.github.com'
          git remote set-url origin https://x-access-token:${{ secrets.GITHUB_TOKEN }}@github.com/${{ github.repository }}
          git add -A
          git commit -m "[Bot] Adding reports from run of Apalache version ${{ env.VERSION }}"
          git push
      - name: Deploy updated site
        # ONLY RUN IF ON TRUNK
        if: github.ref == 'refs/heads/trunk'
        uses: peaceiris/actions-gh-pages@v3
        with:
          github_token: ${{ secrets.GITHUB_TOKEN }}
          publish_dir: ./src/site