ci-results-repo.yml

name: CI - separate results repo
on:
  push:
    branches:
      - master
  pull_request:
    branches:
      - master

jobs:
  benchmarkdotnet-framework:
    name: Run Benchmark.Net .Net Benchmark Framework example - github.com/benchmark-action/github-action-benchmark-results
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - uses: actions/setup-dotnet@v3
        with:
          dotnet-version: '6.0.101'  # SDK Version to use. keep in line with examples/benchmarkdotnet/global.json
      - uses: actions/setup-node@v4
        with:
          node-version: 20
          cache: 'npm'
      - run: npm ci
      - run: npm run build
      - name: Run benchmark
        run: cd examples/benchmarkdotnet && dotnet run --exporters json --filter '*'

      - uses: actions/checkout@v4
        with:
          repository: benchmark-action/github-action-benchmark-results
          ref: 'gh-pages'
          path: 'dist/other-repo'
      - name: Save previous data.js
        run: |
          cp ./dist/other-repo/dev/bench/data.js before_data.js

      - name: Store benchmark result
        uses: ./
        with:
          name: Benchmark.Net Benchmark
          tool: 'benchmarkdotnet'
          output-file-path: examples/benchmarkdotnet/BenchmarkDotNet.Artifacts/results/Sample.Benchmarks-report-full-compressed.json
          fail-on-alert: true
          gh-repository: 'github.com/benchmark-action/github-action-benchmark-results'
          github-token: ${{ secrets.GITHUB_TOKEN }}
      - run: node ./dist/scripts/ci_validate_modification.js before_data.js 'Benchmark.Net Benchmark' './benchmark-data-repository'

  benchmarkjs:
    name: Run JavaScript benchmark example - github.com/benchmark-action/github-action-benchmark-results
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - uses: actions/setup-node@v4
        with:
          node-version: 20
          cache: 'npm'
      - run: npm ci
      - run: npm run build
      - name: Run benchmark
        run: cd examples/benchmarkjs && npm install && node bench.js | tee output.txt

      - uses: actions/checkout@v4
        with:
          repository: benchmark-action/github-action-benchmark-results
          ref: 'gh-pages'
          path: 'dist/other-repo'
      - name: Save previous data.js
        run: |
          cp ./dist/other-repo/dev/bench/data.js before_data.js

      - name: Store benchmark result
        uses: ./
        with:
          name: Benchmark.js Benchmark
          tool: 'benchmarkjs'
          output-file-path: examples/benchmarkjs/output.txt
          fail-on-alert: true
          gh-repository: 'github.com/benchmark-action/github-action-benchmark-results'
          github-token: ${{ secrets.GITHUB_TOKEN }}
      - run: node ./dist/scripts/ci_validate_modification.js before_data.js 'Benchmark.js Benchmark' './benchmark-data-repository'

  catch2-v2-framework:
    name: Run Catch2 C++ Benchmark Framework example (v2.x) - github.com/benchmark-action/github-action-benchmark-results
    runs-on: ubuntu-20.04
    steps:
      - uses: actions/checkout@v4
      - uses: actions/setup-node@v4
        with:
          node-version: 20
          cache: 'npm'
      - run: npm ci
      - run: npm run build
      - name: Run benchmark
        run: |
          cd examples/catch2_v2
          mkdir build && cd build
          cmake -DCMAKE_BUILD_TYPE=Release ..
          cmake --build . --config Release
          ./Catch2_bench > ../benchmark_result.txt

      - uses: actions/checkout@v4
        with:
          repository: benchmark-action/github-action-benchmark-results
          ref: 'gh-pages'
          path: 'dist/other-repo'
      - name: Save previous data.js
        run: |
          cp ./dist/other-repo/dev/bench/data.js before_data.js

      - name: Store benchmark result
        uses: ./
        with:
          name: Catch2 Benchmark
          tool: 'catch2'
          output-file-path: examples/catch2_v2/benchmark_result.txt
          fail-on-alert: true
          gh-repository: 'github.com/benchmark-action/github-action-benchmark-results'
          github-token: ${{ secrets.GITHUB_TOKEN }}
      - run: node ./dist/scripts/ci_validate_modification.js before_data.js 'Catch2 Benchmark' './benchmark-data-repository'

  catch2-v3-framework:
    name: Run Catch2 C++ Benchmark Framework example (v3.x) - github.com/benchmark-action/github-action-benchmark-results
    runs-on: ubuntu-20.04
    steps:
      - uses: actions/checkout@v4
      - uses: actions/setup-node@v4
        with:
          node-version: 20
          cache: 'npm'
      - run: npm ci
      - run: npm run build
      - name: Run benchmark
        run: |
          cd examples/catch2
          mkdir build && cd build
          cmake -DCMAKE_BUILD_TYPE=Release ..
          cmake --build . --config Release
          ./Catch2_bench > ../benchmark_result.txt

      - uses: actions/checkout@v4
        with:
          repository: benchmark-action/github-action-benchmark-results
          ref: 'gh-pages'
          path: 'dist/other-repo'
      - name: Save previous data.js
        run: |
          cp ./dist/other-repo/dev/bench/data.js before_data.js

      - name: Store benchmark result
        uses: ./
        with:
          name: Catch2 Benchmark (v3)
          tool: 'catch2'
          output-file-path: examples/catch2/benchmark_result.txt
          fail-on-alert: true
          gh-repository: 'github.com/benchmark-action/github-action-benchmark-results'
          github-token: ${{ secrets.GITHUB_TOKEN }}
      - run: node ./dist/scripts/ci_validate_modification.js before_data.js 'Catch2 Benchmark (v3)' './benchmark-data-repository'

  cpp-framework:
    name: Run Google C++ Benchmark Framework example - github.com/benchmark-action/github-action-benchmark-results
    runs-on: ubuntu-20.04
    steps:
      - uses: actions/checkout@v4
      - uses: actions/setup-node@v4
        with:
          node-version: 20
          cache: 'npm'
      - run: npm ci
      - run: npm run build
      - name: Cache Benchmark library
        uses: actions/cache@v4
        with:
          path: examples/cpp/benchmark
          key: ${{ runner.os }}-googlebenchmark-v1.5.0
      - name: Run benchmark
        run: |
          cd examples/cpp
          make json

      - uses: actions/checkout@v4
        with:
          repository: benchmark-action/github-action-benchmark-results
          ref: 'gh-pages'
          path: 'dist/other-repo'
      - name: Save previous data.js
        run: |
          cp ./dist/other-repo/dev/bench/data.js before_data.js

      - name: Store benchmark result
        uses: ./
        with:
          name: C++ Benchmark
          tool: 'googlecpp'
          output-file-path: examples/cpp/benchmark_result.json
          fail-on-alert: true
          gh-repository: 'github.com/benchmark-action/github-action-benchmark-results'
          github-token: ${{ secrets.GITHUB_TOKEN }}
      - run: node ./dist/scripts/ci_validate_modification.js before_data.js 'C++ Benchmark' './benchmark-data-repository'

  go:
    name: Run Go benchmark example - github.com/benchmark-action/github-action-benchmark-results
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - uses: actions/setup-node@v4
        with:
          node-version: 20
          cache: 'npm'
      - uses: actions/setup-go@v4
        with:
          go-version: "stable"
      - run: npm ci
      - run: npm run build
      - name: Run benchmark
        run: cd examples/go && go test -bench 'BenchmarkFib' | tee output.txt

      - uses: actions/checkout@v4
        with:
          repository: benchmark-action/github-action-benchmark-results
          ref: 'gh-pages'
          path: 'dist/other-repo'
      - name: Save previous data.js
        run: |
          cp ./dist/other-repo/dev/bench/data.js before_data.js

      - name: Store benchmark result
        uses: ./
        with:
          name: Go Benchmark
          tool: 'go'
          output-file-path: examples/go/output.txt
          fail-on-alert: true
          gh-repository: 'github.com/benchmark-action/github-action-benchmark-results'
          github-token: ${{ secrets.GITHUB_TOKEN }}
      - run: node ./dist/scripts/ci_validate_modification.js before_data.js 'Go Benchmark' './benchmark-data-repository'

  java-jmh:
    name: Run JMH Java Benchmark Framework example
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - uses: actions/setup-node@v4
        with:
          node-version: 20
          cache: 'npm'
      - uses: actions/setup-java@v4
        with:
          distribution: 'adopt'
          java-version: '11'
      - run: npm ci
      - run: npm run build
      - name: Run benchmark
        run: |
          cd examples/java
          mvn clean verify
          java -jar target/benchmarks.jar -wi 1 -i 3 -f 1 -rf json
      - uses: actions/checkout@v4
        with:
          repository: benchmark-action/github-action-benchmark-results
          ref: 'gh-pages'
          path: 'dist/other-repo'
      - name: Save previous data.js
        run: |
          cp ./dist/other-repo/dev/bench/data.js before_data.js
      - name: Store benchmark result
        uses: ./
        with:
          name: JMH Benchmark
          tool: "jmh"
          output-file-path: examples/java/jmh-result.json
          fail-on-alert: true
          gh-repository: 'github.com/benchmark-action/github-action-benchmark-results'
          github-token: ${{ secrets.GITHUB_TOKEN }}
      - run: node ./dist/scripts/ci_validate_modification.js before_data.js 'JMH Benchmark' './benchmark-data-repository'

  julia-benchmark:
    name: Run Julia benchmark example - github.com/benchmark-action/github-action-benchmark-results
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - uses: actions/setup-node@v4
        with:
          node-version: 20
          cache: 'npm'
      - run: npm ci
      - run: npm run build
      - uses: julia-actions/setup-julia@v1
        with:
          version: '1'
          arch: x64
      - name: Run benchmark
        run: |
          cd examples/julia
          julia --project --color=yes -e '
            using Pkg;
            Pkg.instantiate();
            include("fib.jl")'

      - uses: actions/checkout@v4
        with:
          repository: benchmark-action/github-action-benchmark-results
          ref: 'gh-pages'
          path: 'dist/other-repo'
      - name: Save previous data.js
        run: |
          cp ./dist/other-repo/dev/bench/data.js before_data.js

      - name: Store benchmark result
        uses: ./
        with:
          name: Julia benchmark
          tool: 'julia'
          output-file-path: examples/julia/output.json
          fail-on-alert: true
          gh-repository: 'github.com/benchmark-action/github-action-benchmark-results'
          github-token: ${{ secrets.GITHUB_TOKEN }}
      - run: node ./dist/scripts/ci_validate_modification.js before_data.js 'Julia benchmark' './benchmark-data-repository'

  pytest-benchmark:
    name: Run Pytest benchmark example - github.com/benchmark-action/github-action-benchmark-results
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - uses: actions/setup-node@v4
        with:
          node-version: 20
          cache: 'npm'
      - uses: actions/setup-python@v5
        with:
          python-version: 3.9
      - run: npm ci
      - run: npm run build
      - name: Run benchmark
        run: |
          cd examples/pytest
          pip install -r requirements.txt
          pytest bench.py --benchmark-json output.json

      - uses: actions/checkout@v4
        with:
          repository: benchmark-action/github-action-benchmark-results
          ref: 'gh-pages'
          path: 'dist/other-repo'
      - name: Save previous data.js
        run: |
          cp ./dist/other-repo/dev/bench/data.js before_data.js

      - name: Store benchmark result
        uses: ./
        with:
          name: Python Benchmark with pytest-benchmark
          tool: 'pytest'
          output-file-path: examples/pytest/output.json
          fail-on-alert: true
          gh-repository: 'github.com/benchmark-action/github-action-benchmark-results'
          github-token: ${{ secrets.GITHUB_TOKEN }}
      - run: node ./dist/scripts/ci_validate_modification.js before_data.js 'Python Benchmark with pytest-benchmark' './benchmark-data-repository'

  rust:
    name: Run Rust benchmark example - github.com/benchmark-action/github-action-benchmark-results
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - uses: actions/setup-node@v4
        with:
          node-version: 20
          cache: 'npm'
      - run: npm ci
      - run: npm run build
      - run: rustup toolchain update nightly && rustup default nightly
      - name: Run benchmark
        run: cd examples/rust && cargo +nightly bench | tee output.txt

      - uses: actions/checkout@v4
        with:
          repository: benchmark-action/github-action-benchmark-results
          ref: 'gh-pages'
          path: 'dist/other-repo'
      - name: Save previous data.js
        run: |
          cp ./dist/other-repo/dev/bench/data.js before_data.js

      - name: Store benchmark result
        uses: ./
        with:
          name: Rust Benchmark
          tool: 'cargo'
          output-file-path: examples/rust/output.txt
          fail-on-alert: true
          gh-repository: 'github.com/benchmark-action/github-action-benchmark-results'
          github-token: ${{ secrets.GITHUB_TOKEN }}
      - run: node ./dist/scripts/ci_validate_modification.js before_data.js 'Rust Benchmark' './benchmark-data-repository'

  rust-criterion-rs-framework:
    name: Run Criterion.rs benchmark example - github.com/benchmark-action/github-action-benchmark-results
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - uses: actions/setup-node@v4
        with:
          node-version: 20
          cache: 'npm'
      - run: npm ci
      - run: npm run build
      - run: rustup toolchain update nightly && rustup default nightly
      - name: Run benchmark
        run: cd examples/criterion-rs && cargo +nightly bench -- --output-format bencher | tee output.txt

      - uses: actions/checkout@v4
        with:
          repository: benchmark-action/github-action-benchmark-results
          ref: 'gh-pages'
          path: 'dist/other-repo'
      - name: Save previous data.js
        run: |
          cp ./dist/other-repo/dev/bench/data.js before_data.js

      - name: Store benchmark result
        uses: ./
        with:
          name: Criterion.rs Benchmark
          tool: 'cargo'
          output-file-path: examples/criterion-rs/output.txt
          fail-on-alert: true
          gh-repository: 'github.com/benchmark-action/github-action-benchmark-results'
          github-token: ${{ secrets.GITHUB_TOKEN }}
      - run: node ./dist/scripts/ci_validate_modification.js before_data.js 'Criterion.rs Benchmark' './benchmark-data-repository'

  only-alert-with-cache:
    name: Run alert check with actions/cache
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - uses: actions/setup-node@v4
        with:
          node-version: 20
          cache: 'npm'
      - run: npm ci
      - run: npm run build
      - uses: actions/setup-go@v4
        with:
          go-version: "stable"
      - name: Run benchmark
        run: cd examples/go && go test -bench 'BenchmarkFib' | tee output.txt
      - name: Download previous benchmark data
        uses: actions/cache@v4
        with:
          path: ./cache
          key: ${{ runner.os }}-ci-cache-test
      - name: Store benchmark result
        uses: ./
        with:
          name: Alert setup example with cache
          tool: 'go'
          output-file-path: examples/go/output.txt
          external-data-json-path: ./cache/benchmark-data.json
          github-token: ${{ secrets.GITHUB_TOKEN }}
          alert-threshold: '200%'
          comment-on-alert: true
          fail-on-alert: true
          alert-comment-cc-users: '@ktrz'