benchmark.yml

name: 'benchmark (pull request)'

on:
  pull_request:
    branches: [ main ]

jobs:
  fetch-branch:
    name: 'Fetch branch'
    runs-on: ubuntu-latest

    steps:
    - id: identify
      run: echo "##[set-output name=branch;]$(echo ${GITHUB_HEAD_REF#refs/heads/})"

    outputs:
      branch: ${{ steps.identify.outputs.branch }}

  fetch-remote:
    name: 'Fetch remote'
    runs-on: ubuntu-latest

    steps:
      - id: identify
        run: |
          echo "##[set-output name=remote;]$(if [ ${GITHUB_REPOSITORY} == 'SSoelvsten/adiar' ]; then echo 'origin'; else echo '${GITHUB_REPOSITORY}'; fi)"

    outputs:
      remote: ${{ steps.identify.outputs.remote }}

  skip_duplicate:
    name: 'Skip Job?'
    continue-on-error: true
    runs-on: ubuntu-latest

    steps:
      - id: skip_check
        uses: fkirc/skip-duplicate-actions@master
        with:
          paths: '["src/adiar/**"]'

    outputs:
      should_skip: ${{ steps.skip_check.outputs.should_skip }}

  run_picotrav:
    name: 'Performance Regression: Picotrav ${{ matrix.circuit }}'
    runs-on: ubuntu-latest

    needs: [fetch-branch, fetch-remote, skip_duplicate]
    if: ${{ needs.skip_duplicate.outputs.should_skip != 'true' }}

    strategy:
      fail-fast: false
      matrix:
        include:
          - { circuit: adder }
          - { circuit: arbiter }
          - { circuit: mem_ctrl }

    steps:
      - uses: actions/checkout@v2
        with:
          repository: 'ssoelvsten/bdd-benchmark'
          submodules: 'recursive'
          fetch-depth: '0'

      - name: 'apt install'
        run: |
          sudo apt update
          sudo apt install libboost-all-dev

      - name: 'pip install'
        run: |
          sudo pip install wget

      - name: 'python3 regression.py'
        run: |
          python3 ./regression.py <<< "picotrav
          yes
          ${{ matrix.circuit }}
          adiar
          8192
          origin
          main
          ${{ needs.fetch-remote.outputs.remote }}
          ${{ needs.fetch-branch.outputs.branch }}
          no
          no
          3
          10
          "

      - name: 'post regression_adiar.out'
        if: always()
        uses: mshick/add-pr-comment@main
        with:
          message-path: ./regression_adiar.out
          message-id: 'benchmark_picotrav_${{ matrix.circuit }}'

  run_qbf:
    name: 'Performance Regression: QBF ${{ matrix.category }}/${{ matrix.circuit }}'
    runs-on: ubuntu-latest

    needs: [fetch-branch, fetch-remote, skip_duplicate]
    if: ${{ needs.skip_duplicate.outputs.should_skip != 'true' }}

    strategy:
      fail-fast: false
      matrix:
        include:
          - { category: breakthrough,      circuit: 3x4_19_bwnib }
          - { category: breakthrough_dual, circuit: 3x6_10_bwnib }
          - { category: connect4,          circuit: 6x6_11_connect4_bwnib }
          - { category: domineering,       circuit: 5x5_13_bwnib }
          - { category: ep,                circuit: 8x8_7_e-8-1_p-3-4_bwnib }
          - { category: ep_dual,           circuit: 8x8_6_e-8-1_p-2-3_bwnib }
          - { category: hex,               circuit: hein_08_5x5-11_bwnib }
          - { category: httt,              circuit: 4x4_9_tippy_bwnib }

    steps:
      - uses: actions/checkout@v2
        with:
          repository: 'ssoelvsten/bdd-benchmark'
          submodules: 'recursive'
          fetch-depth: '0'

      - name: 'apt install'
        run: |
          sudo apt update
          sudo apt install libboost-all-dev

      - name: 'pip install'
        run: |
          sudo pip install wget

      - name: 'python3 regression.py'
        run: |
          python3 ./regression.py <<< "qbf
          yes
          ${{ matrix.category }}
          ${{ matrix.circuit }}
          adiar
          8192
          origin
          main
          ${{ needs.fetch-remote.outputs.remote }}
          ${{ needs.fetch-branch.outputs.branch }}
          no
          no
          3
          10
          "

      - name: 'post regression_adiar.out'
        if: always()
        uses: mshick/add-pr-comment@main
        with:
          message-path: ./regression_adiar.out
          message-id: 'benchmark_qbf_${{ matrix.circuit }}'

  run_queens:
    name: 'Performance Regression: ${{ matrix.N }}-Queens'
    runs-on: ubuntu-latest

    needs: [fetch-branch, fetch-remote, skip_duplicate]
    if: ${{ needs.skip_duplicate.outputs.should_skip != 'true' }}

    strategy:
      fail-fast: false
      matrix:
        include:
          - { N: 12 }
          - { N: 14 }

    steps:
      - uses: actions/checkout@v2
        with:
          repository: 'ssoelvsten/bdd-benchmark'
          submodules: 'recursive'
          fetch-depth: '0'

      - name: 'apt install'
        run: |
          sudo apt update
          sudo apt install libboost-all-dev

      - name: 'pip install'
        run: |
          sudo pip install wget

      - name: 'python3 regression.py'
        run: |
          python3 ./regression.py <<< "queens
          ${{ matrix.N }}
          adiar
          8192
          origin
          main
          ${{ needs.fetch-remote.outputs.remote }}
          ${{ needs.fetch-branch.outputs.branch }}
          no
          no
          3
          10
          "

      - name: 'post regression_adiar.out'
        if: always()
        uses: mshick/add-pr-comment@main
        with:
          message-path: ./regression_adiar.out
          message-id: 'benchmark_queens_${{ matrix.N }}'

  run_dummy:
    name: 'Performance Regression: ${{ matrix.title }}'
    runs-on: ubuntu-latest

    needs: [skip_duplicate]
    if: ${{ needs.skip_duplicate.outputs.should_skip == 'true' }}

    strategy:
      matrix:
        include:
          - title: 'Picotrav adder'
          - title: 'Picotrav arbiter'
          - title: 'Picotrav mem_ctrl'
          - title: '12-Queens'
          - title: '14-Queens'
          - title: 'QBF breakthrough/3x4_19_bwnib'
          - title: 'QBF breakthrough_dual/3x6_10_bwnib'
          - title: 'QBF connect4/6x6_11_connect4_bwnib'
          - title: 'QBF domineering/5x5_13_bwnib'
          - title: 'QBF ep/8x8_7_e-8-1_p-3-4_bwnib'
          - title: 'QBF ep_dual/8x8_6_e-8-1_p-2-3_bwnib'
          - title: 'QBF hex/hein_08_5x5-11_bwnib'
          - title: 'QBF httt/4x4_9_tippy_bwnib'

    steps:
    - name: Echo skip
      run: |
        echo "Running benchmarks is skipped"