py-bench.yml

name: py-bench

on:
  pull_request:
    paths:
      - "python/langsmith/**"

env:
  POETRY_VERSION: "1.7.1"

jobs:
  benchmark:
    runs-on: ubuntu-latest
    defaults:
      run:
        working-directory: python
    steps:
      - uses: actions/checkout@v4
      - id: files
        name: Get changed files
        uses: Ana06/get-changed-files@v2.3.0
        with:
          format: json
      - name: Set up Python 3.11 + Poetry ${{ env.POETRY_VERSION }}
        uses: "./.github/actions/poetry_setup"
        with:
          python-version: "3.11"
          poetry-version: ${{ env.POETRY_VERSION }}
          cache-key: py-bench
      - name: Install dependencies
        run: poetry install --with dev
      - name: Download baseline
        uses: actions/cache/restore@v4
        with:
          key: ${{ runner.os }}-benchmark-baseline
          restore-keys: |
            ${{ runner.os }}-benchmark-baseline-
          fail-on-cache-miss: true
          path: |
            python/out/benchmark-baseline.json
      - name: Run benchmarks
        id: benchmark
        run: |
          {
            echo 'OUTPUT<<EOF'
            make -s benchmark-fast
            echo EOF
          } >> "$GITHUB_OUTPUT"
      - name: Compare benchmarks
        id: compare
        run: |
          {
            echo 'OUTPUT<<EOF'
            mv out/benchmark-baseline.json out/main.json
            mv out/benchmark.json out/changes.json
            poetry run pyperf compare_to out/main.json out/changes.json --table --group-by-speed
            echo EOF
          } >> "$GITHUB_OUTPUT"
      - name: Annotation
        uses: actions/github-script@v7
        with:
          script: |
            const file = JSON.parse(`${{ steps.files.outputs.added_modified_renamed }}`)[0]
            core.notice(`${{ steps.benchmark.outputs.OUTPUT }}`, {
              title: 'Benchmark results',
              file,
            })
            core.notice(`${{ steps.compare.outputs.OUTPUT }}`, {
              title: 'Comparison against main',
              file,
            })