From 019cbcbff5b46768d07df85029eb19c76f835b75 Mon Sep 17 00:00:00 2001 From: Chris Trzesniewski Date: Tue, 26 Mar 2024 09:21:10 +0100 Subject: [PATCH 1/8] fix: 231 rust bench not comparing to baseline investigate --- .github/workflows/ci-results-repo.yml | 608 +++++++++++------------ .github/workflows/ci.yml | 684 +++++++++++++------------- .gitignore | 1 + src/write.ts | 1 + 4 files changed, 648 insertions(+), 646 deletions(-) diff --git a/.github/workflows/ci-results-repo.yml b/.github/workflows/ci-results-repo.yml index 7358f7d8a..0245c9fb8 100644 --- a/.github/workflows/ci-results-repo.yml +++ b/.github/workflows/ci-results-repo.yml @@ -43,307 +43,307 @@ jobs: github-token: ${{ secrets.GITHUB_TOKEN }} - run: node ./dist/scripts/ci_validate_modification.js before_data.js 'Rust Benchmark' './benchmark-data-repository' - go: - name: Run Go benchmark example - github.com/benchmark-action/github-action-benchmark-results - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-node@v4 - with: - node-version: 20 - cache: 'npm' - - uses: actions/setup-go@v4 - with: - go-version: "stable" - - run: npm ci - - run: npm run build - - name: Run benchmark - run: cd examples/go && go test -bench 'BenchmarkFib' | tee output.txt - - - uses: actions/checkout@v4 - with: - repository: benchmark-action/github-action-benchmark-results - ref: 'gh-pages' - path: 'dist/other-repo' - - name: Save previous data.js - run: | - cp ./dist/other-repo/dev/bench/data.js before_data.js - - - name: Store benchmark result - uses: ./ - with: - name: Go Benchmark - tool: 'go' - output-file-path: examples/go/output.txt - fail-on-alert: true - gh-repository: 'github.com/benchmark-action/github-action-benchmark-results' - github-token: ${{ secrets.GITHUB_TOKEN }} - - run: node ./dist/scripts/ci_validate_modification.js before_data.js 'Go Benchmark' './benchmark-data-repository' - - benchmarkjs: - name: Run JavaScript benchmark example - github.com/benchmark-action/github-action-benchmark-results - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-node@v4 - with: - node-version: 20 - cache: 'npm' - - run: npm ci - - run: npm run build - - name: Run benchmark - run: cd examples/benchmarkjs && npm install && node bench.js | tee output.txt - - - uses: actions/checkout@v4 - with: - repository: benchmark-action/github-action-benchmark-results - ref: 'gh-pages' - path: 'dist/other-repo' - - name: Save previous data.js - run: | - cp ./dist/other-repo/dev/bench/data.js before_data.js - - - name: Store benchmark result - uses: ./ - with: - name: Benchmark.js Benchmark - tool: 'benchmarkjs' - output-file-path: examples/benchmarkjs/output.txt - fail-on-alert: true - gh-repository: 'github.com/benchmark-action/github-action-benchmark-results' - github-token: ${{ secrets.GITHUB_TOKEN }} - - run: node ./dist/scripts/ci_validate_modification.js before_data.js 'Benchmark.js Benchmark' './benchmark-data-repository' - - pytest-benchmark: - name: Run Pytest benchmark example - github.com/benchmark-action/github-action-benchmark-results - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-node@v4 - with: - node-version: 20 - cache: 'npm' - - uses: actions/setup-python@v5 - with: - python-version: 3.9 - - run: npm ci - - run: npm run build - - name: Run benchmark - run: | - cd examples/pytest - pip install -r requirements.txt - pytest bench.py --benchmark-json output.json - - - uses: actions/checkout@v4 - with: - repository: benchmark-action/github-action-benchmark-results - ref: 'gh-pages' - path: 'dist/other-repo' - - name: Save previous data.js - run: | - cp ./dist/other-repo/dev/bench/data.js before_data.js - - - name: Store benchmark result - uses: ./ - with: - name: Python Benchmark with pytest-benchmark - tool: 'pytest' - output-file-path: examples/pytest/output.json - fail-on-alert: true - gh-repository: 'github.com/benchmark-action/github-action-benchmark-results' - github-token: ${{ secrets.GITHUB_TOKEN }} - - run: node ./dist/scripts/ci_validate_modification.js before_data.js 'Python Benchmark with pytest-benchmark' './benchmark-data-repository' - - google-benchmark-framework: - name: Run Google C++ Benchmark Framework example - github.com/benchmark-action/github-action-benchmark-results - runs-on: ubuntu-20.04 - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-node@v4 - with: - node-version: 20 - cache: 'npm' - - run: npm ci - - run: npm run build - - name: Cache Benchmark library - uses: actions/cache@v4 - with: - path: examples/cpp/benchmark - key: ${{ runner.os }}-googlebenchmark-v1.5.0 - - name: Run benchmark - run: | - cd examples/cpp - make json - - - uses: actions/checkout@v4 - with: - repository: benchmark-action/github-action-benchmark-results - ref: 'gh-pages' - path: 'dist/other-repo' - - name: Save previous data.js - run: | - cp ./dist/other-repo/dev/bench/data.js before_data.js - - - name: Store benchmark result - uses: ./ - with: - name: C++ Benchmark - tool: 'googlecpp' - output-file-path: examples/cpp/benchmark_result.json - fail-on-alert: true - gh-repository: 'github.com/benchmark-action/github-action-benchmark-results' - github-token: ${{ secrets.GITHUB_TOKEN }} - - run: node ./dist/scripts/ci_validate_modification.js before_data.js 'C++ Benchmark' './benchmark-data-repository' - - catch2-framework: - name: Run Catch2 C++ Benchmark Framework example - github.com/benchmark-action/github-action-benchmark-results - runs-on: ubuntu-20.04 - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-node@v4 - with: - node-version: 20 - cache: 'npm' - - run: npm ci - - run: npm run build - - name: Run benchmark - run: | - cd examples/catch2 - mkdir build && cd build - cmake -DCMAKE_BUILD_TYPE=Release .. - cmake --build . --config Release - ./Catch2_bench > ../benchmark_result.txt - - - uses: actions/checkout@v4 - with: - repository: benchmark-action/github-action-benchmark-results - ref: 'gh-pages' - path: 'dist/other-repo' - - name: Save previous data.js - run: | - cp ./dist/other-repo/dev/bench/data.js before_data.js - - - name: Store benchmark result - uses: ./ - with: - name: Catch2 Benchmark - tool: 'catch2' - output-file-path: examples/catch2/benchmark_result.txt - fail-on-alert: true - gh-repository: 'github.com/benchmark-action/github-action-benchmark-results' - github-token: ${{ secrets.GITHUB_TOKEN }} - - run: node ./dist/scripts/ci_validate_modification.js before_data.js 'Catch2 Benchmark' './benchmark-data-repository' - - julia-benchmark: - name: Run Julia benchmark example - github.com/benchmark-action/github-action-benchmark-results - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-node@v4 - with: - node-version: 20 - cache: 'npm' - - run: npm ci - - run: npm run build - - uses: julia-actions/setup-julia@v1 - with: - version: '1' - arch: x64 - - name: Run benchmark - run: | - cd examples/julia - julia --project --color=yes -e ' - using Pkg; - Pkg.instantiate(); - include("fib.jl")' - - - uses: actions/checkout@v4 - with: - repository: benchmark-action/github-action-benchmark-results - ref: 'gh-pages' - path: 'dist/other-repo' - - name: Save previous data.js - run: | - cp ./dist/other-repo/dev/bench/data.js before_data.js - - - name: Store benchmark result - uses: ./ - with: - name: Julia benchmark - tool: 'julia' - output-file-path: examples/julia/output.json - fail-on-alert: true - gh-repository: 'github.com/benchmark-action/github-action-benchmark-results' - github-token: ${{ secrets.GITHUB_TOKEN }} - - run: node ./dist/scripts/ci_validate_modification.js before_data.js 'Julia benchmark' './benchmark-data-repository' - - benchmarkdotnet-framework: - name: Run Benchmark.Net .Net Benchmark Framework example - github.com/benchmark-action/github-action-benchmark-results - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-dotnet@v3 - with: - dotnet-version: '6.0.101' # SDK Version to use. keep in line with examples/benchmarkdotnet/global.json - - uses: actions/setup-node@v4 - with: - node-version: 20 - cache: 'npm' - - run: npm ci - - run: npm run build - - name: Run benchmark - run: cd examples/benchmarkdotnet && dotnet run --exporters json --filter '*' - - - uses: actions/checkout@v4 - with: - repository: benchmark-action/github-action-benchmark-results - ref: 'gh-pages' - path: 'dist/other-repo' - - name: Save previous data.js - run: | - cp ./dist/other-repo/dev/bench/data.js before_data.js - - - name: Store benchmark result - uses: ./ - with: - name: Benchmark.Net Benchmark - tool: 'benchmarkdotnet' - output-file-path: examples/benchmarkdotnet/BenchmarkDotNet.Artifacts/results/Sample.Benchmarks-report-full-compressed.json - fail-on-alert: true - gh-repository: 'github.com/benchmark-action/github-action-benchmark-results' - github-token: ${{ secrets.GITHUB_TOKEN }} - - run: node ./dist/scripts/ci_validate_modification.js before_data.js 'Benchmark.Net Benchmark' './benchmark-data-repository' - - only-alert-with-cache: - name: Run alert check with actions/cache - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-node@v4 - with: - node-version: 20 - cache: 'npm' - - run: npm ci - - run: npm run build - - uses: actions/setup-go@v4 - with: - go-version: "stable" - - name: Run benchmark - run: cd examples/go && go test -bench 'BenchmarkFib' | tee output.txt - - name: Download previous benchmark data - uses: actions/cache@v4 - with: - path: ./cache - key: ${{ runner.os }}-ci-cache-test - - name: Store benchmark result - uses: ./ - with: - name: Alert setup example with cache - tool: 'go' - output-file-path: examples/go/output.txt - external-data-json-path: ./cache/benchmark-data.json - github-token: ${{ secrets.GITHUB_TOKEN }} - alert-threshold: '200%' - comment-on-alert: true - fail-on-alert: true - alert-comment-cc-users: '@ktrz' +# go: +# name: Run Go benchmark example - github.com/benchmark-action/github-action-benchmark-results +# runs-on: ubuntu-latest +# steps: +# - uses: actions/checkout@v4 +# - uses: actions/setup-node@v4 +# with: +# node-version: 20 +# cache: 'npm' +# - uses: actions/setup-go@v4 +# with: +# go-version: "stable" +# - run: npm ci +# - run: npm run build +# - name: Run benchmark +# run: cd examples/go && go test -bench 'BenchmarkFib' | tee output.txt +# +# - uses: actions/checkout@v4 +# with: +# repository: benchmark-action/github-action-benchmark-results +# ref: 'gh-pages' +# path: 'dist/other-repo' +# - name: Save previous data.js +# run: | +# cp ./dist/other-repo/dev/bench/data.js before_data.js +# +# - name: Store benchmark result +# uses: ./ +# with: +# name: Go Benchmark +# tool: 'go' +# output-file-path: examples/go/output.txt +# fail-on-alert: true +# gh-repository: 'github.com/benchmark-action/github-action-benchmark-results' +# github-token: ${{ secrets.GITHUB_TOKEN }} +# - run: node ./dist/scripts/ci_validate_modification.js before_data.js 'Go Benchmark' './benchmark-data-repository' +# +# benchmarkjs: +# name: Run JavaScript benchmark example - github.com/benchmark-action/github-action-benchmark-results +# runs-on: ubuntu-latest +# steps: +# - uses: actions/checkout@v4 +# - uses: actions/setup-node@v4 +# with: +# node-version: 20 +# cache: 'npm' +# - run: npm ci +# - run: npm run build +# - name: Run benchmark +# run: cd examples/benchmarkjs && npm install && node bench.js | tee output.txt +# +# - uses: actions/checkout@v4 +# with: +# repository: benchmark-action/github-action-benchmark-results +# ref: 'gh-pages' +# path: 'dist/other-repo' +# - name: Save previous data.js +# run: | +# cp ./dist/other-repo/dev/bench/data.js before_data.js +# +# - name: Store benchmark result +# uses: ./ +# with: +# name: Benchmark.js Benchmark +# tool: 'benchmarkjs' +# output-file-path: examples/benchmarkjs/output.txt +# fail-on-alert: true +# gh-repository: 'github.com/benchmark-action/github-action-benchmark-results' +# github-token: ${{ secrets.GITHUB_TOKEN }} +# - run: node ./dist/scripts/ci_validate_modification.js before_data.js 'Benchmark.js Benchmark' './benchmark-data-repository' +# +# pytest-benchmark: +# name: Run Pytest benchmark example - github.com/benchmark-action/github-action-benchmark-results +# runs-on: ubuntu-latest +# steps: +# - uses: actions/checkout@v4 +# - uses: actions/setup-node@v4 +# with: +# node-version: 20 +# cache: 'npm' +# - uses: actions/setup-python@v5 +# with: +# python-version: 3.9 +# - run: npm ci +# - run: npm run build +# - name: Run benchmark +# run: | +# cd examples/pytest +# pip install -r requirements.txt +# pytest bench.py --benchmark-json output.json +# +# - uses: actions/checkout@v4 +# with: +# repository: benchmark-action/github-action-benchmark-results +# ref: 'gh-pages' +# path: 'dist/other-repo' +# - name: Save previous data.js +# run: | +# cp ./dist/other-repo/dev/bench/data.js before_data.js +# +# - name: Store benchmark result +# uses: ./ +# with: +# name: Python Benchmark with pytest-benchmark +# tool: 'pytest' +# output-file-path: examples/pytest/output.json +# fail-on-alert: true +# gh-repository: 'github.com/benchmark-action/github-action-benchmark-results' +# github-token: ${{ secrets.GITHUB_TOKEN }} +# - run: node ./dist/scripts/ci_validate_modification.js before_data.js 'Python Benchmark with pytest-benchmark' './benchmark-data-repository' +# +# google-benchmark-framework: +# name: Run Google C++ Benchmark Framework example - github.com/benchmark-action/github-action-benchmark-results +# runs-on: ubuntu-20.04 +# steps: +# - uses: actions/checkout@v4 +# - uses: actions/setup-node@v4 +# with: +# node-version: 20 +# cache: 'npm' +# - run: npm ci +# - run: npm run build +# - name: Cache Benchmark library +# uses: actions/cache@v4 +# with: +# path: examples/cpp/benchmark +# key: ${{ runner.os }}-googlebenchmark-v1.5.0 +# - name: Run benchmark +# run: | +# cd examples/cpp +# make json +# +# - uses: actions/checkout@v4 +# with: +# repository: benchmark-action/github-action-benchmark-results +# ref: 'gh-pages' +# path: 'dist/other-repo' +# - name: Save previous data.js +# run: | +# cp ./dist/other-repo/dev/bench/data.js before_data.js +# +# - name: Store benchmark result +# uses: ./ +# with: +# name: C++ Benchmark +# tool: 'googlecpp' +# output-file-path: examples/cpp/benchmark_result.json +# fail-on-alert: true +# gh-repository: 'github.com/benchmark-action/github-action-benchmark-results' +# github-token: ${{ secrets.GITHUB_TOKEN }} +# - run: node ./dist/scripts/ci_validate_modification.js before_data.js 'C++ Benchmark' './benchmark-data-repository' +# +# catch2-framework: +# name: Run Catch2 C++ Benchmark Framework example - github.com/benchmark-action/github-action-benchmark-results +# runs-on: ubuntu-20.04 +# steps: +# - uses: actions/checkout@v4 +# - uses: actions/setup-node@v4 +# with: +# node-version: 20 +# cache: 'npm' +# - run: npm ci +# - run: npm run build +# - name: Run benchmark +# run: | +# cd examples/catch2 +# mkdir build && cd build +# cmake -DCMAKE_BUILD_TYPE=Release .. +# cmake --build . --config Release +# ./Catch2_bench > ../benchmark_result.txt +# +# - uses: actions/checkout@v4 +# with: +# repository: benchmark-action/github-action-benchmark-results +# ref: 'gh-pages' +# path: 'dist/other-repo' +# - name: Save previous data.js +# run: | +# cp ./dist/other-repo/dev/bench/data.js before_data.js +# +# - name: Store benchmark result +# uses: ./ +# with: +# name: Catch2 Benchmark +# tool: 'catch2' +# output-file-path: examples/catch2/benchmark_result.txt +# fail-on-alert: true +# gh-repository: 'github.com/benchmark-action/github-action-benchmark-results' +# github-token: ${{ secrets.GITHUB_TOKEN }} +# - run: node ./dist/scripts/ci_validate_modification.js before_data.js 'Catch2 Benchmark' './benchmark-data-repository' +# +# julia-benchmark: +# name: Run Julia benchmark example - github.com/benchmark-action/github-action-benchmark-results +# runs-on: ubuntu-latest +# steps: +# - uses: actions/checkout@v4 +# - uses: actions/setup-node@v4 +# with: +# node-version: 20 +# cache: 'npm' +# - run: npm ci +# - run: npm run build +# - uses: julia-actions/setup-julia@v1 +# with: +# version: '1' +# arch: x64 +# - name: Run benchmark +# run: | +# cd examples/julia +# julia --project --color=yes -e ' +# using Pkg; +# Pkg.instantiate(); +# include("fib.jl")' +# +# - uses: actions/checkout@v4 +# with: +# repository: benchmark-action/github-action-benchmark-results +# ref: 'gh-pages' +# path: 'dist/other-repo' +# - name: Save previous data.js +# run: | +# cp ./dist/other-repo/dev/bench/data.js before_data.js +# +# - name: Store benchmark result +# uses: ./ +# with: +# name: Julia benchmark +# tool: 'julia' +# output-file-path: examples/julia/output.json +# fail-on-alert: true +# gh-repository: 'github.com/benchmark-action/github-action-benchmark-results' +# github-token: ${{ secrets.GITHUB_TOKEN }} +# - run: node ./dist/scripts/ci_validate_modification.js before_data.js 'Julia benchmark' './benchmark-data-repository' +# +# benchmarkdotnet-framework: +# name: Run Benchmark.Net .Net Benchmark Framework example - github.com/benchmark-action/github-action-benchmark-results +# runs-on: ubuntu-latest +# steps: +# - uses: actions/checkout@v4 +# - uses: actions/setup-dotnet@v3 +# with: +# dotnet-version: '6.0.101' # SDK Version to use. keep in line with examples/benchmarkdotnet/global.json +# - uses: actions/setup-node@v4 +# with: +# node-version: 20 +# cache: 'npm' +# - run: npm ci +# - run: npm run build +# - name: Run benchmark +# run: cd examples/benchmarkdotnet && dotnet run --exporters json --filter '*' +# +# - uses: actions/checkout@v4 +# with: +# repository: benchmark-action/github-action-benchmark-results +# ref: 'gh-pages' +# path: 'dist/other-repo' +# - name: Save previous data.js +# run: | +# cp ./dist/other-repo/dev/bench/data.js before_data.js +# +# - name: Store benchmark result +# uses: ./ +# with: +# name: Benchmark.Net Benchmark +# tool: 'benchmarkdotnet' +# output-file-path: examples/benchmarkdotnet/BenchmarkDotNet.Artifacts/results/Sample.Benchmarks-report-full-compressed.json +# fail-on-alert: true +# gh-repository: 'github.com/benchmark-action/github-action-benchmark-results' +# github-token: ${{ secrets.GITHUB_TOKEN }} +# - run: node ./dist/scripts/ci_validate_modification.js before_data.js 'Benchmark.Net Benchmark' './benchmark-data-repository' +# +# only-alert-with-cache: +# name: Run alert check with actions/cache +# runs-on: ubuntu-latest +# steps: +# - uses: actions/checkout@v4 +# - uses: actions/setup-node@v4 +# with: +# node-version: 20 +# cache: 'npm' +# - run: npm ci +# - run: npm run build +# - uses: actions/setup-go@v4 +# with: +# go-version: "stable" +# - name: Run benchmark +# run: cd examples/go && go test -bench 'BenchmarkFib' | tee output.txt +# - name: Download previous benchmark data +# uses: actions/cache@v4 +# with: +# path: ./cache +# key: ${{ runner.os }}-ci-cache-test +# - name: Store benchmark result +# uses: ./ +# with: +# name: Alert setup example with cache +# tool: 'go' +# output-file-path: examples/go/output.txt +# external-data-json-path: ./cache/benchmark-data.json +# github-token: ${{ secrets.GITHUB_TOKEN }} +# alert-threshold: '200%' +# comment-on-alert: true +# fail-on-alert: true +# alert-comment-cc-users: '@ktrz' diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index eed878cb8..35a258560 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -38,345 +38,345 @@ jobs: summary-always: true - run: node ./dist/scripts/ci_validate_modification.js before_data.js 'Rust Benchmark' - go: - name: Run Go benchmark example - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-node@v4 - with: - node-version: 20 - cache: 'npm' - - uses: actions/setup-go@v4 - with: - go-version: "stable" - - run: npm ci - - run: npm run build - - name: Run benchmark - run: cd examples/go && go test -bench 'BenchmarkFib' | tee output.txt - - name: Save previous data.js - run: | - git fetch origin gh-pages - git checkout gh-pages - cp ./dev/bench/data.js before_data.js - git checkout - - - name: Store benchmark result - uses: ./ - with: - name: Go Benchmark - tool: 'go' - output-file-path: examples/go/output.txt - skip-fetch-gh-pages: true - fail-on-alert: true - summary-always: true - - run: node ./dist/scripts/ci_validate_modification.js before_data.js 'Go Benchmark' - - benchmarkjs: - name: Run JavaScript benchmark example - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-node@v4 - with: - node-version: 20 - cache: 'npm' - - run: npm ci - - run: npm run build - - name: Run benchmark - run: cd examples/benchmarkjs && npm install && node bench.js | tee output.txt - - name: Save previous data.js - run: | - git fetch origin gh-pages - git checkout gh-pages - cp ./dev/bench/data.js before_data.js - git checkout - - - name: Store benchmark result - uses: ./ - with: - name: Benchmark.js Benchmark - tool: 'benchmarkjs' - output-file-path: examples/benchmarkjs/output.txt - skip-fetch-gh-pages: true - fail-on-alert: true - summary-always: true - - run: node ./dist/scripts/ci_validate_modification.js before_data.js 'Benchmark.js Benchmark' - - pytest-benchmark: - name: Run Pytest benchmark example - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-node@v4 - with: - node-version: 20 - cache: 'npm' - - uses: actions/setup-python@v5 - with: - python-version: 3.9 - - run: npm ci - - run: npm run build - - name: Run benchmark - run: | - cd examples/pytest - pip install -r requirements.txt - pytest bench.py --benchmark-json output.json - - name: Save previous data.js - run: | - git fetch origin gh-pages - git checkout gh-pages - cp ./dev/bench/data.js before_data.js - git checkout - - - name: Store benchmark result - uses: ./ - with: - name: Python Benchmark with pytest-benchmark - tool: 'pytest' - output-file-path: examples/pytest/output.json - skip-fetch-gh-pages: true - fail-on-alert: true - summary-always: true - - run: node ./dist/scripts/ci_validate_modification.js before_data.js 'Python Benchmark with pytest-benchmark' - - google-benchmark-framework: - name: Run Google C++ Benchmark Framework example - runs-on: ubuntu-20.04 - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-node@v4 - with: - node-version: 20 - cache: 'npm' - - run: npm ci - - run: npm run build - - name: Cache Benchmark library - uses: actions/cache@v4 - with: - path: examples/cpp/benchmark - key: ${{ runner.os }}-googlebenchmark-v1.5.0 - - name: Run benchmark - run: | - cd examples/cpp - make json - - name: Save previous data.js - run: | - git fetch origin gh-pages - git checkout gh-pages - cp ./dev/bench/data.js before_data.js - git checkout - - - name: Store benchmark result - uses: ./ - with: - name: C++ Benchmark - tool: 'googlecpp' - output-file-path: examples/cpp/benchmark_result.json - skip-fetch-gh-pages: true - fail-on-alert: true - summary-always: true - - run: node ./dist/scripts/ci_validate_modification.js before_data.js 'C++ Benchmark' - - catch2-framework: - name: Run Catch2 C++ Benchmark Framework example - runs-on: ubuntu-20.04 - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-node@v4 - with: - node-version: 20 - cache: 'npm' - - run: npm ci - - run: npm run build - - name: Run benchmark - run: | - cd examples/catch2 - mkdir build && cd build - cmake -DCMAKE_BUILD_TYPE=Release .. - cmake --build . --config Release - ./Catch2_bench > ../benchmark_result.txt - - name: Save previous data.js - run: | - git fetch origin gh-pages - git checkout gh-pages - cp ./dev/bench/data.js before_data.js - git checkout - - - name: Store benchmark result - uses: ./ - with: - name: Catch2 Benchmark - tool: 'catch2' - output-file-path: examples/catch2/benchmark_result.txt - skip-fetch-gh-pages: true - fail-on-alert: true - summary-always: true - - run: node ./dist/scripts/ci_validate_modification.js before_data.js 'Catch2 Benchmark' - - julia-benchmark: - name: Run Julia benchmark example - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-node@v4 - with: - node-version: 20 - cache: 'npm' - - run: npm ci - - run: npm run build - - uses: julia-actions/setup-julia@v1 - with: - version: '1' - arch: x64 - - name: Run benchmark - run: | - cd examples/julia - julia --project --color=yes -e ' - using Pkg; - Pkg.instantiate(); - include("fib.jl")' - - name: Save previous data.js - run: | - git fetch origin gh-pages - git checkout gh-pages - cp ./dev/bench/data.js before_data.js - git checkout - - - name: Store benchmark result - uses: ./ - with: - name: Julia benchmark - tool: 'julia' - output-file-path: examples/julia/output.json - skip-fetch-gh-pages: true - fail-on-alert: true - summary-always: true - - run: node ./dist/scripts/ci_validate_modification.js before_data.js 'Julia benchmark' - - benchmarkdotnet-framework: - name: Run Benchmark.Net .Net Benchmark Framework example - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-node@v4 - with: - node-version: 20 - cache: 'npm' - - uses: actions/setup-dotnet@v3 - with: - dotnet-version: '6.0.101' # SDK Version to use. keep in line with examples/benchmarkdotnet/global.json - - run: npm ci - - run: npm run build - - name: Run benchmark - run: cd examples/benchmarkdotnet && dotnet run --exporters json --filter '*' - - name: Save previous data.js - run: | - git fetch origin gh-pages - git checkout gh-pages - cp ./dev/bench/data.js before_data.js - git checkout - - - name: Store benchmark result - uses: ./ - with: - name: Benchmark.Net Benchmark - tool: 'benchmarkdotnet' - output-file-path: examples/benchmarkdotnet/BenchmarkDotNet.Artifacts/results/Sample.Benchmarks-report-full-compressed.json - skip-fetch-gh-pages: true - fail-on-alert: true - summary-always: true - - run: node ./dist/scripts/ci_validate_modification.js before_data.js 'Benchmark.Net Benchmark' - - jmh: - name: Run JMH Java Benchmark Framework example - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-node@v4 - with: - node-version: 20 - cache: 'npm' - - uses: actions/setup-java@v4 - with: - distribution: 'adopt' - java-version: '11' - - run: npm ci - - run: npm run build - - name: Run benchmark - run: | - cd examples/java - mvn clean verify - java -jar target/benchmarks.jar -wi 1 -i 3 -f 1 -rf json - - name: Save previous data.js - run: | - git fetch origin gh-pages - git checkout gh-pages - cp ./dev/bench/data.js before_data.js - git checkout - - - name: Store benchmark result - uses: ./ - with: - name: JMH Benchmark - tool: "jmh" - output-file-path: examples/java/jmh-result.json - skip-fetch-gh-pages: true - fail-on-alert: true - summary-always: true - - run: node ./dist/scripts/ci_validate_modification.js before_data.js 'JMH Benchmark' - - only-alert-with-cache: - name: Run alert check with actions/cache - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-node@v4 - with: - node-version: 20 - cache: 'npm' - - run: npm ci - - run: npm run build - - uses: actions/setup-go@v4 - with: - go-version: "stable" - - name: Run benchmark - run: cd examples/go && go test -bench 'BenchmarkFib' | tee output.txt - - name: Download previous benchmark data - uses: actions/cache@v4 - with: - path: ./cache - key: ${{ runner.os }}-ci-cache-test - - name: Store benchmark result - uses: ./ - with: - name: Alert setup example with cache - tool: 'go' - output-file-path: examples/go/output.txt - external-data-json-path: ./cache/benchmark-data.json - github-token: ${{ secrets.GITHUB_TOKEN }} - alert-threshold: '200%' - comment-on-alert: true - fail-on-alert: true - alert-comment-cc-users: '@ktrz' - unit-tests: - name: Run unit tests - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-node@v4 - with: - node-version: 20 - cache: 'npm' - - run: npm ci - - run: npm run coverage - - name: Apply yamllint - run: | - sudo pip install yamllint - yamllint --strict .github/workflows - - uses: codecov/codecov-action@v2 - - lint: - name: Run linting and formatting check - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-node@v4 - with: - node-version: 20 - cache: 'npm' - - run: npm ci - - run: npm run lint - - run: npm run format:check +# go: +# name: Run Go benchmark example +# runs-on: ubuntu-latest +# steps: +# - uses: actions/checkout@v4 +# - uses: actions/setup-node@v4 +# with: +# node-version: 20 +# cache: 'npm' +# - uses: actions/setup-go@v4 +# with: +# go-version: "stable" +# - run: npm ci +# - run: npm run build +# - name: Run benchmark +# run: cd examples/go && go test -bench 'BenchmarkFib' | tee output.txt +# - name: Save previous data.js +# run: | +# git fetch origin gh-pages +# git checkout gh-pages +# cp ./dev/bench/data.js before_data.js +# git checkout - +# - name: Store benchmark result +# uses: ./ +# with: +# name: Go Benchmark +# tool: 'go' +# output-file-path: examples/go/output.txt +# skip-fetch-gh-pages: true +# fail-on-alert: true +# summary-always: true +# - run: node ./dist/scripts/ci_validate_modification.js before_data.js 'Go Benchmark' +# +# benchmarkjs: +# name: Run JavaScript benchmark example +# runs-on: ubuntu-latest +# steps: +# - uses: actions/checkout@v4 +# - uses: actions/setup-node@v4 +# with: +# node-version: 20 +# cache: 'npm' +# - run: npm ci +# - run: npm run build +# - name: Run benchmark +# run: cd examples/benchmarkjs && npm install && node bench.js | tee output.txt +# - name: Save previous data.js +# run: | +# git fetch origin gh-pages +# git checkout gh-pages +# cp ./dev/bench/data.js before_data.js +# git checkout - +# - name: Store benchmark result +# uses: ./ +# with: +# name: Benchmark.js Benchmark +# tool: 'benchmarkjs' +# output-file-path: examples/benchmarkjs/output.txt +# skip-fetch-gh-pages: true +# fail-on-alert: true +# summary-always: true +# - run: node ./dist/scripts/ci_validate_modification.js before_data.js 'Benchmark.js Benchmark' +# +# pytest-benchmark: +# name: Run Pytest benchmark example +# runs-on: ubuntu-latest +# steps: +# - uses: actions/checkout@v4 +# - uses: actions/setup-node@v4 +# with: +# node-version: 20 +# cache: 'npm' +# - uses: actions/setup-python@v5 +# with: +# python-version: 3.9 +# - run: npm ci +# - run: npm run build +# - name: Run benchmark +# run: | +# cd examples/pytest +# pip install -r requirements.txt +# pytest bench.py --benchmark-json output.json +# - name: Save previous data.js +# run: | +# git fetch origin gh-pages +# git checkout gh-pages +# cp ./dev/bench/data.js before_data.js +# git checkout - +# - name: Store benchmark result +# uses: ./ +# with: +# name: Python Benchmark with pytest-benchmark +# tool: 'pytest' +# output-file-path: examples/pytest/output.json +# skip-fetch-gh-pages: true +# fail-on-alert: true +# summary-always: true +# - run: node ./dist/scripts/ci_validate_modification.js before_data.js 'Python Benchmark with pytest-benchmark' +# +# google-benchmark-framework: +# name: Run Google C++ Benchmark Framework example +# runs-on: ubuntu-20.04 +# steps: +# - uses: actions/checkout@v4 +# - uses: actions/setup-node@v4 +# with: +# node-version: 20 +# cache: 'npm' +# - run: npm ci +# - run: npm run build +# - name: Cache Benchmark library +# uses: actions/cache@v4 +# with: +# path: examples/cpp/benchmark +# key: ${{ runner.os }}-googlebenchmark-v1.5.0 +# - name: Run benchmark +# run: | +# cd examples/cpp +# make json +# - name: Save previous data.js +# run: | +# git fetch origin gh-pages +# git checkout gh-pages +# cp ./dev/bench/data.js before_data.js +# git checkout - +# - name: Store benchmark result +# uses: ./ +# with: +# name: C++ Benchmark +# tool: 'googlecpp' +# output-file-path: examples/cpp/benchmark_result.json +# skip-fetch-gh-pages: true +# fail-on-alert: true +# summary-always: true +# - run: node ./dist/scripts/ci_validate_modification.js before_data.js 'C++ Benchmark' +# +# catch2-framework: +# name: Run Catch2 C++ Benchmark Framework example +# runs-on: ubuntu-20.04 +# steps: +# - uses: actions/checkout@v4 +# - uses: actions/setup-node@v4 +# with: +# node-version: 20 +# cache: 'npm' +# - run: npm ci +# - run: npm run build +# - name: Run benchmark +# run: | +# cd examples/catch2 +# mkdir build && cd build +# cmake -DCMAKE_BUILD_TYPE=Release .. +# cmake --build . --config Release +# ./Catch2_bench > ../benchmark_result.txt +# - name: Save previous data.js +# run: | +# git fetch origin gh-pages +# git checkout gh-pages +# cp ./dev/bench/data.js before_data.js +# git checkout - +# - name: Store benchmark result +# uses: ./ +# with: +# name: Catch2 Benchmark +# tool: 'catch2' +# output-file-path: examples/catch2/benchmark_result.txt +# skip-fetch-gh-pages: true +# fail-on-alert: true +# summary-always: true +# - run: node ./dist/scripts/ci_validate_modification.js before_data.js 'Catch2 Benchmark' +# +# julia-benchmark: +# name: Run Julia benchmark example +# runs-on: ubuntu-latest +# steps: +# - uses: actions/checkout@v4 +# - uses: actions/setup-node@v4 +# with: +# node-version: 20 +# cache: 'npm' +# - run: npm ci +# - run: npm run build +# - uses: julia-actions/setup-julia@v1 +# with: +# version: '1' +# arch: x64 +# - name: Run benchmark +# run: | +# cd examples/julia +# julia --project --color=yes -e ' +# using Pkg; +# Pkg.instantiate(); +# include("fib.jl")' +# - name: Save previous data.js +# run: | +# git fetch origin gh-pages +# git checkout gh-pages +# cp ./dev/bench/data.js before_data.js +# git checkout - +# - name: Store benchmark result +# uses: ./ +# with: +# name: Julia benchmark +# tool: 'julia' +# output-file-path: examples/julia/output.json +# skip-fetch-gh-pages: true +# fail-on-alert: true +# summary-always: true +# - run: node ./dist/scripts/ci_validate_modification.js before_data.js 'Julia benchmark' +# +# benchmarkdotnet-framework: +# name: Run Benchmark.Net .Net Benchmark Framework example +# runs-on: ubuntu-latest +# steps: +# - uses: actions/checkout@v4 +# - uses: actions/setup-node@v4 +# with: +# node-version: 20 +# cache: 'npm' +# - uses: actions/setup-dotnet@v3 +# with: +# dotnet-version: '6.0.101' # SDK Version to use. keep in line with examples/benchmarkdotnet/global.json +# - run: npm ci +# - run: npm run build +# - name: Run benchmark +# run: cd examples/benchmarkdotnet && dotnet run --exporters json --filter '*' +# - name: Save previous data.js +# run: | +# git fetch origin gh-pages +# git checkout gh-pages +# cp ./dev/bench/data.js before_data.js +# git checkout - +# - name: Store benchmark result +# uses: ./ +# with: +# name: Benchmark.Net Benchmark +# tool: 'benchmarkdotnet' +# output-file-path: examples/benchmarkdotnet/BenchmarkDotNet.Artifacts/results/Sample.Benchmarks-report-full-compressed.json +# skip-fetch-gh-pages: true +# fail-on-alert: true +# summary-always: true +# - run: node ./dist/scripts/ci_validate_modification.js before_data.js 'Benchmark.Net Benchmark' +# +# jmh: +# name: Run JMH Java Benchmark Framework example +# runs-on: ubuntu-latest +# steps: +# - uses: actions/checkout@v4 +# - uses: actions/setup-node@v4 +# with: +# node-version: 20 +# cache: 'npm' +# - uses: actions/setup-java@v4 +# with: +# distribution: 'adopt' +# java-version: '11' +# - run: npm ci +# - run: npm run build +# - name: Run benchmark +# run: | +# cd examples/java +# mvn clean verify +# java -jar target/benchmarks.jar -wi 1 -i 3 -f 1 -rf json +# - name: Save previous data.js +# run: | +# git fetch origin gh-pages +# git checkout gh-pages +# cp ./dev/bench/data.js before_data.js +# git checkout - +# - name: Store benchmark result +# uses: ./ +# with: +# name: JMH Benchmark +# tool: "jmh" +# output-file-path: examples/java/jmh-result.json +# skip-fetch-gh-pages: true +# fail-on-alert: true +# summary-always: true +# - run: node ./dist/scripts/ci_validate_modification.js before_data.js 'JMH Benchmark' +# +# only-alert-with-cache: +# name: Run alert check with actions/cache +# runs-on: ubuntu-latest +# steps: +# - uses: actions/checkout@v4 +# - uses: actions/setup-node@v4 +# with: +# node-version: 20 +# cache: 'npm' +# - run: npm ci +# - run: npm run build +# - uses: actions/setup-go@v4 +# with: +# go-version: "stable" +# - name: Run benchmark +# run: cd examples/go && go test -bench 'BenchmarkFib' | tee output.txt +# - name: Download previous benchmark data +# uses: actions/cache@v4 +# with: +# path: ./cache +# key: ${{ runner.os }}-ci-cache-test +# - name: Store benchmark result +# uses: ./ +# with: +# name: Alert setup example with cache +# tool: 'go' +# output-file-path: examples/go/output.txt +# external-data-json-path: ./cache/benchmark-data.json +# github-token: ${{ secrets.GITHUB_TOKEN }} +# alert-threshold: '200%' +# comment-on-alert: true +# fail-on-alert: true +# alert-comment-cc-users: '@ktrz' +# unit-tests: +# name: Run unit tests +# runs-on: ubuntu-latest +# steps: +# - uses: actions/checkout@v4 +# - uses: actions/setup-node@v4 +# with: +# node-version: 20 +# cache: 'npm' +# - run: npm ci +# - run: npm run coverage +# - name: Apply yamllint +# run: | +# sudo pip install yamllint +# yamllint --strict .github/workflows +# - uses: codecov/codecov-action@v2 +# +# lint: +# name: Run linting and formatting check +# runs-on: ubuntu-latest +# steps: +# - uses: actions/checkout@v4 +# - uses: actions/setup-node@v4 +# with: +# node-version: 20 +# cache: 'npm' +# - run: npm ci +# - run: npm run lint +# - run: npm run format:check diff --git a/.gitignore b/.gitignore index c6a69e261..9578ebeea 100644 --- a/.gitignore +++ b/.gitignore @@ -12,3 +12,4 @@ /.idea target/ jmh-result.json +.env diff --git a/src/write.ts b/src/write.ts index 3c71823a7..d774454e8 100644 --- a/src/write.ts +++ b/src/write.ts @@ -94,6 +94,7 @@ interface Alert { function findAlerts(curSuite: Benchmark, prevSuite: Benchmark, threshold: number): Alert[] { core.debug(`Comparing current:${curSuite.commit.id} and prev:${prevSuite.commit.id} for alert`); + core.debug(`Previous suite benches: ${JSON.stringify(prevSuite.benches)}`); const alerts = []; for (const current of curSuite.benches) { From 2897372aa29b550a76d98d750b9fc440afe01dfe Mon Sep 17 00:00:00 2001 From: Chris Trzesniewski Date: Tue, 26 Mar 2024 09:52:15 +0100 Subject: [PATCH 2/8] fix: 231 rust bench not comparing to baseline sort jobs in ci workflows add missing benches to ci jobs rename Criterion bench to distinguish between default Rust and Criterion benchmarks --- .github/workflows/ci-results-repo.yml | 677 +++++++++-------- .github/workflows/ci.yml | 708 +++++++++--------- .github/workflows/cpp.yml | 2 +- ...criterion-rs.yml => rust-criterion-rs.yml} | 4 +- 4 files changed, 742 insertions(+), 649 deletions(-) rename .github/workflows/{criterion-rs.yml => rust-criterion-rs.yml} (95%) diff --git a/.github/workflows/ci-results-repo.yml b/.github/workflows/ci-results-repo.yml index 0245c9fb8..27ec9f6f4 100644 --- a/.github/workflows/ci-results-repo.yml +++ b/.github/workflows/ci-results-repo.yml @@ -8,6 +8,317 @@ on: - master jobs: + benchmarkdotnet-framework: + name: Run Benchmark.Net .Net Benchmark Framework example - github.com/benchmark-action/github-action-benchmark-results + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-dotnet@v3 + with: + dotnet-version: '6.0.101' # SDK Version to use. keep in line with examples/benchmarkdotnet/global.json + - uses: actions/setup-node@v4 + with: + node-version: 20 + cache: 'npm' + - run: npm ci + - run: npm run build + - name: Run benchmark + run: cd examples/benchmarkdotnet && dotnet run --exporters json --filter '*' + + - uses: actions/checkout@v4 + with: + repository: benchmark-action/github-action-benchmark-results + ref: 'gh-pages' + path: 'dist/other-repo' + - name: Save previous data.js + run: | + cp ./dist/other-repo/dev/bench/data.js before_data.js + + - name: Store benchmark result + uses: ./ + with: + name: Benchmark.Net Benchmark + tool: 'benchmarkdotnet' + output-file-path: examples/benchmarkdotnet/BenchmarkDotNet.Artifacts/results/Sample.Benchmarks-report-full-compressed.json + fail-on-alert: true + gh-repository: 'github.com/benchmark-action/github-action-benchmark-results' + github-token: ${{ secrets.GITHUB_TOKEN }} + - run: node ./dist/scripts/ci_validate_modification.js before_data.js 'Benchmark.Net Benchmark' './benchmark-data-repository' + + benchmarkjs: + name: Run JavaScript benchmark example - github.com/benchmark-action/github-action-benchmark-results + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version: 20 + cache: 'npm' + - run: npm ci + - run: npm run build + - name: Run benchmark + run: cd examples/benchmarkjs && npm install && node bench.js | tee output.txt + + - uses: actions/checkout@v4 + with: + repository: benchmark-action/github-action-benchmark-results + ref: 'gh-pages' + path: 'dist/other-repo' + - name: Save previous data.js + run: | + cp ./dist/other-repo/dev/bench/data.js before_data.js + + - name: Store benchmark result + uses: ./ + with: + name: Benchmark.js Benchmark + tool: 'benchmarkjs' + output-file-path: examples/benchmarkjs/output.txt + fail-on-alert: true + gh-repository: 'github.com/benchmark-action/github-action-benchmark-results' + github-token: ${{ secrets.GITHUB_TOKEN }} + - run: node ./dist/scripts/ci_validate_modification.js before_data.js 'Benchmark.js Benchmark' './benchmark-data-repository' + + catch2-framework: + name: Run Catch2 C++ Benchmark Framework example - github.com/benchmark-action/github-action-benchmark-results + runs-on: ubuntu-20.04 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version: 20 + cache: 'npm' + - run: npm ci + - run: npm run build + - name: Run benchmark + run: | + cd examples/catch2 + mkdir build && cd build + cmake -DCMAKE_BUILD_TYPE=Release .. + cmake --build . --config Release + ./Catch2_bench > ../benchmark_result.txt + + - uses: actions/checkout@v4 + with: + repository: benchmark-action/github-action-benchmark-results + ref: 'gh-pages' + path: 'dist/other-repo' + - name: Save previous data.js + run: | + cp ./dist/other-repo/dev/bench/data.js before_data.js + + - name: Store benchmark result + uses: ./ + with: + name: Catch2 Benchmark + tool: 'catch2' + output-file-path: examples/catch2/benchmark_result.txt + fail-on-alert: true + gh-repository: 'github.com/benchmark-action/github-action-benchmark-results' + github-token: ${{ secrets.GITHUB_TOKEN }} + - run: node ./dist/scripts/ci_validate_modification.js before_data.js 'Catch2 Benchmark' './benchmark-data-repository' + + cpp-framework: + name: Run Google C++ Benchmark Framework example - github.com/benchmark-action/github-action-benchmark-results + runs-on: ubuntu-20.04 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version: 20 + cache: 'npm' + - run: npm ci + - run: npm run build + - name: Cache Benchmark library + uses: actions/cache@v4 + with: + path: examples/cpp/benchmark + key: ${{ runner.os }}-googlebenchmark-v1.5.0 + - name: Run benchmark + run: | + cd examples/cpp + make json + + - uses: actions/checkout@v4 + with: + repository: benchmark-action/github-action-benchmark-results + ref: 'gh-pages' + path: 'dist/other-repo' + - name: Save previous data.js + run: | + cp ./dist/other-repo/dev/bench/data.js before_data.js + + - name: Store benchmark result + uses: ./ + with: + name: C++ Benchmark + tool: 'googlecpp' + output-file-path: examples/cpp/benchmark_result.json + fail-on-alert: true + gh-repository: 'github.com/benchmark-action/github-action-benchmark-results' + github-token: ${{ secrets.GITHUB_TOKEN }} + - run: node ./dist/scripts/ci_validate_modification.js before_data.js 'C++ Benchmark' './benchmark-data-repository' + + go: + name: Run Go benchmark example - github.com/benchmark-action/github-action-benchmark-results + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version: 20 + cache: 'npm' + - uses: actions/setup-go@v4 + with: + go-version: "stable" + - run: npm ci + - run: npm run build + - name: Run benchmark + run: cd examples/go && go test -bench 'BenchmarkFib' | tee output.txt + + - uses: actions/checkout@v4 + with: + repository: benchmark-action/github-action-benchmark-results + ref: 'gh-pages' + path: 'dist/other-repo' + - name: Save previous data.js + run: | + cp ./dist/other-repo/dev/bench/data.js before_data.js + + - name: Store benchmark result + uses: ./ + with: + name: Go Benchmark + tool: 'go' + output-file-path: examples/go/output.txt + fail-on-alert: true + gh-repository: 'github.com/benchmark-action/github-action-benchmark-results' + github-token: ${{ secrets.GITHUB_TOKEN }} + - run: node ./dist/scripts/ci_validate_modification.js before_data.js 'Go Benchmark' './benchmark-data-repository' + + java-jmh: + name: Run JMH Java Benchmark Framework example + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version: 20 + cache: 'npm' + - uses: actions/setup-java@v4 + with: + distribution: 'adopt' + java-version: '11' + - run: npm ci + - run: npm run build + - name: Run benchmark + run: | + cd examples/java + mvn clean verify + java -jar target/benchmarks.jar -wi 1 -i 3 -f 1 -rf json + - uses: actions/checkout@v4 + with: + repository: benchmark-action/github-action-benchmark-results + ref: 'gh-pages' + path: 'dist/other-repo' + - name: Save previous data.js + run: | + cp ./dist/other-repo/dev/bench/data.js before_data.js + - name: Store benchmark result + uses: ./ + with: + name: JMH Benchmark + tool: "jmh" + output-file-path: examples/java/jmh-result.json + skip-fetch-gh-pages: true + fail-on-alert: true + gh-repository: 'github.com/benchmark-action/github-action-benchmark-results' + github-token: ${{ secrets.GITHUB_TOKEN }} + - run: node ./dist/scripts/ci_validate_modification.js before_data.js 'JMH Benchmark' + + julia-benchmark: + name: Run Julia benchmark example - github.com/benchmark-action/github-action-benchmark-results + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version: 20 + cache: 'npm' + - run: npm ci + - run: npm run build + - uses: julia-actions/setup-julia@v1 + with: + version: '1' + arch: x64 + - name: Run benchmark + run: | + cd examples/julia + julia --project --color=yes -e ' + using Pkg; + Pkg.instantiate(); + include("fib.jl")' + + - uses: actions/checkout@v4 + with: + repository: benchmark-action/github-action-benchmark-results + ref: 'gh-pages' + path: 'dist/other-repo' + - name: Save previous data.js + run: | + cp ./dist/other-repo/dev/bench/data.js before_data.js + + - name: Store benchmark result + uses: ./ + with: + name: Julia benchmark + tool: 'julia' + output-file-path: examples/julia/output.json + fail-on-alert: true + gh-repository: 'github.com/benchmark-action/github-action-benchmark-results' + github-token: ${{ secrets.GITHUB_TOKEN }} + - run: node ./dist/scripts/ci_validate_modification.js before_data.js 'Julia benchmark' './benchmark-data-repository' + + pytest-benchmark: + name: Run Pytest benchmark example - github.com/benchmark-action/github-action-benchmark-results + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version: 20 + cache: 'npm' + - uses: actions/setup-python@v5 + with: + python-version: 3.9 + - run: npm ci + - run: npm run build + - name: Run benchmark + run: | + cd examples/pytest + pip install -r requirements.txt + pytest bench.py --benchmark-json output.json + + - uses: actions/checkout@v4 + with: + repository: benchmark-action/github-action-benchmark-results + ref: 'gh-pages' + path: 'dist/other-repo' + - name: Save previous data.js + run: | + cp ./dist/other-repo/dev/bench/data.js before_data.js + + - name: Store benchmark result + uses: ./ + with: + name: Python Benchmark with pytest-benchmark + tool: 'pytest' + output-file-path: examples/pytest/output.json + fail-on-alert: true + gh-repository: 'github.com/benchmark-action/github-action-benchmark-results' + github-token: ${{ secrets.GITHUB_TOKEN }} + - run: node ./dist/scripts/ci_validate_modification.js before_data.js 'Python Benchmark with pytest-benchmark' './benchmark-data-repository' + rust: name: Run Rust benchmark example - github.com/benchmark-action/github-action-benchmark-results runs-on: ubuntu-latest @@ -43,307 +354,65 @@ jobs: github-token: ${{ secrets.GITHUB_TOKEN }} - run: node ./dist/scripts/ci_validate_modification.js before_data.js 'Rust Benchmark' './benchmark-data-repository' -# go: -# name: Run Go benchmark example - github.com/benchmark-action/github-action-benchmark-results -# runs-on: ubuntu-latest -# steps: -# - uses: actions/checkout@v4 -# - uses: actions/setup-node@v4 -# with: -# node-version: 20 -# cache: 'npm' -# - uses: actions/setup-go@v4 -# with: -# go-version: "stable" -# - run: npm ci -# - run: npm run build -# - name: Run benchmark -# run: cd examples/go && go test -bench 'BenchmarkFib' | tee output.txt -# -# - uses: actions/checkout@v4 -# with: -# repository: benchmark-action/github-action-benchmark-results -# ref: 'gh-pages' -# path: 'dist/other-repo' -# - name: Save previous data.js -# run: | -# cp ./dist/other-repo/dev/bench/data.js before_data.js -# -# - name: Store benchmark result -# uses: ./ -# with: -# name: Go Benchmark -# tool: 'go' -# output-file-path: examples/go/output.txt -# fail-on-alert: true -# gh-repository: 'github.com/benchmark-action/github-action-benchmark-results' -# github-token: ${{ secrets.GITHUB_TOKEN }} -# - run: node ./dist/scripts/ci_validate_modification.js before_data.js 'Go Benchmark' './benchmark-data-repository' -# -# benchmarkjs: -# name: Run JavaScript benchmark example - github.com/benchmark-action/github-action-benchmark-results -# runs-on: ubuntu-latest -# steps: -# - uses: actions/checkout@v4 -# - uses: actions/setup-node@v4 -# with: -# node-version: 20 -# cache: 'npm' -# - run: npm ci -# - run: npm run build -# - name: Run benchmark -# run: cd examples/benchmarkjs && npm install && node bench.js | tee output.txt -# -# - uses: actions/checkout@v4 -# with: -# repository: benchmark-action/github-action-benchmark-results -# ref: 'gh-pages' -# path: 'dist/other-repo' -# - name: Save previous data.js -# run: | -# cp ./dist/other-repo/dev/bench/data.js before_data.js -# -# - name: Store benchmark result -# uses: ./ -# with: -# name: Benchmark.js Benchmark -# tool: 'benchmarkjs' -# output-file-path: examples/benchmarkjs/output.txt -# fail-on-alert: true -# gh-repository: 'github.com/benchmark-action/github-action-benchmark-results' -# github-token: ${{ secrets.GITHUB_TOKEN }} -# - run: node ./dist/scripts/ci_validate_modification.js before_data.js 'Benchmark.js Benchmark' './benchmark-data-repository' -# -# pytest-benchmark: -# name: Run Pytest benchmark example - github.com/benchmark-action/github-action-benchmark-results -# runs-on: ubuntu-latest -# steps: -# - uses: actions/checkout@v4 -# - uses: actions/setup-node@v4 -# with: -# node-version: 20 -# cache: 'npm' -# - uses: actions/setup-python@v5 -# with: -# python-version: 3.9 -# - run: npm ci -# - run: npm run build -# - name: Run benchmark -# run: | -# cd examples/pytest -# pip install -r requirements.txt -# pytest bench.py --benchmark-json output.json -# -# - uses: actions/checkout@v4 -# with: -# repository: benchmark-action/github-action-benchmark-results -# ref: 'gh-pages' -# path: 'dist/other-repo' -# - name: Save previous data.js -# run: | -# cp ./dist/other-repo/dev/bench/data.js before_data.js -# -# - name: Store benchmark result -# uses: ./ -# with: -# name: Python Benchmark with pytest-benchmark -# tool: 'pytest' -# output-file-path: examples/pytest/output.json -# fail-on-alert: true -# gh-repository: 'github.com/benchmark-action/github-action-benchmark-results' -# github-token: ${{ secrets.GITHUB_TOKEN }} -# - run: node ./dist/scripts/ci_validate_modification.js before_data.js 'Python Benchmark with pytest-benchmark' './benchmark-data-repository' -# -# google-benchmark-framework: -# name: Run Google C++ Benchmark Framework example - github.com/benchmark-action/github-action-benchmark-results -# runs-on: ubuntu-20.04 -# steps: -# - uses: actions/checkout@v4 -# - uses: actions/setup-node@v4 -# with: -# node-version: 20 -# cache: 'npm' -# - run: npm ci -# - run: npm run build -# - name: Cache Benchmark library -# uses: actions/cache@v4 -# with: -# path: examples/cpp/benchmark -# key: ${{ runner.os }}-googlebenchmark-v1.5.0 -# - name: Run benchmark -# run: | -# cd examples/cpp -# make json -# -# - uses: actions/checkout@v4 -# with: -# repository: benchmark-action/github-action-benchmark-results -# ref: 'gh-pages' -# path: 'dist/other-repo' -# - name: Save previous data.js -# run: | -# cp ./dist/other-repo/dev/bench/data.js before_data.js -# -# - name: Store benchmark result -# uses: ./ -# with: -# name: C++ Benchmark -# tool: 'googlecpp' -# output-file-path: examples/cpp/benchmark_result.json -# fail-on-alert: true -# gh-repository: 'github.com/benchmark-action/github-action-benchmark-results' -# github-token: ${{ secrets.GITHUB_TOKEN }} -# - run: node ./dist/scripts/ci_validate_modification.js before_data.js 'C++ Benchmark' './benchmark-data-repository' -# -# catch2-framework: -# name: Run Catch2 C++ Benchmark Framework example - github.com/benchmark-action/github-action-benchmark-results -# runs-on: ubuntu-20.04 -# steps: -# - uses: actions/checkout@v4 -# - uses: actions/setup-node@v4 -# with: -# node-version: 20 -# cache: 'npm' -# - run: npm ci -# - run: npm run build -# - name: Run benchmark -# run: | -# cd examples/catch2 -# mkdir build && cd build -# cmake -DCMAKE_BUILD_TYPE=Release .. -# cmake --build . --config Release -# ./Catch2_bench > ../benchmark_result.txt -# -# - uses: actions/checkout@v4 -# with: -# repository: benchmark-action/github-action-benchmark-results -# ref: 'gh-pages' -# path: 'dist/other-repo' -# - name: Save previous data.js -# run: | -# cp ./dist/other-repo/dev/bench/data.js before_data.js -# -# - name: Store benchmark result -# uses: ./ -# with: -# name: Catch2 Benchmark -# tool: 'catch2' -# output-file-path: examples/catch2/benchmark_result.txt -# fail-on-alert: true -# gh-repository: 'github.com/benchmark-action/github-action-benchmark-results' -# github-token: ${{ secrets.GITHUB_TOKEN }} -# - run: node ./dist/scripts/ci_validate_modification.js before_data.js 'Catch2 Benchmark' './benchmark-data-repository' -# -# julia-benchmark: -# name: Run Julia benchmark example - github.com/benchmark-action/github-action-benchmark-results -# runs-on: ubuntu-latest -# steps: -# - uses: actions/checkout@v4 -# - uses: actions/setup-node@v4 -# with: -# node-version: 20 -# cache: 'npm' -# - run: npm ci -# - run: npm run build -# - uses: julia-actions/setup-julia@v1 -# with: -# version: '1' -# arch: x64 -# - name: Run benchmark -# run: | -# cd examples/julia -# julia --project --color=yes -e ' -# using Pkg; -# Pkg.instantiate(); -# include("fib.jl")' -# -# - uses: actions/checkout@v4 -# with: -# repository: benchmark-action/github-action-benchmark-results -# ref: 'gh-pages' -# path: 'dist/other-repo' -# - name: Save previous data.js -# run: | -# cp ./dist/other-repo/dev/bench/data.js before_data.js -# -# - name: Store benchmark result -# uses: ./ -# with: -# name: Julia benchmark -# tool: 'julia' -# output-file-path: examples/julia/output.json -# fail-on-alert: true -# gh-repository: 'github.com/benchmark-action/github-action-benchmark-results' -# github-token: ${{ secrets.GITHUB_TOKEN }} -# - run: node ./dist/scripts/ci_validate_modification.js before_data.js 'Julia benchmark' './benchmark-data-repository' -# -# benchmarkdotnet-framework: -# name: Run Benchmark.Net .Net Benchmark Framework example - github.com/benchmark-action/github-action-benchmark-results -# runs-on: ubuntu-latest -# steps: -# - uses: actions/checkout@v4 -# - uses: actions/setup-dotnet@v3 -# with: -# dotnet-version: '6.0.101' # SDK Version to use. keep in line with examples/benchmarkdotnet/global.json -# - uses: actions/setup-node@v4 -# with: -# node-version: 20 -# cache: 'npm' -# - run: npm ci -# - run: npm run build -# - name: Run benchmark -# run: cd examples/benchmarkdotnet && dotnet run --exporters json --filter '*' -# -# - uses: actions/checkout@v4 -# with: -# repository: benchmark-action/github-action-benchmark-results -# ref: 'gh-pages' -# path: 'dist/other-repo' -# - name: Save previous data.js -# run: | -# cp ./dist/other-repo/dev/bench/data.js before_data.js -# -# - name: Store benchmark result -# uses: ./ -# with: -# name: Benchmark.Net Benchmark -# tool: 'benchmarkdotnet' -# output-file-path: examples/benchmarkdotnet/BenchmarkDotNet.Artifacts/results/Sample.Benchmarks-report-full-compressed.json -# fail-on-alert: true -# gh-repository: 'github.com/benchmark-action/github-action-benchmark-results' -# github-token: ${{ secrets.GITHUB_TOKEN }} -# - run: node ./dist/scripts/ci_validate_modification.js before_data.js 'Benchmark.Net Benchmark' './benchmark-data-repository' -# -# only-alert-with-cache: -# name: Run alert check with actions/cache -# runs-on: ubuntu-latest -# steps: -# - uses: actions/checkout@v4 -# - uses: actions/setup-node@v4 -# with: -# node-version: 20 -# cache: 'npm' -# - run: npm ci -# - run: npm run build -# - uses: actions/setup-go@v4 -# with: -# go-version: "stable" -# - name: Run benchmark -# run: cd examples/go && go test -bench 'BenchmarkFib' | tee output.txt -# - name: Download previous benchmark data -# uses: actions/cache@v4 -# with: -# path: ./cache -# key: ${{ runner.os }}-ci-cache-test -# - name: Store benchmark result -# uses: ./ -# with: -# name: Alert setup example with cache -# tool: 'go' -# output-file-path: examples/go/output.txt -# external-data-json-path: ./cache/benchmark-data.json -# github-token: ${{ secrets.GITHUB_TOKEN }} -# alert-threshold: '200%' -# comment-on-alert: true -# fail-on-alert: true -# alert-comment-cc-users: '@ktrz' + rust-criterion-rs-framework: + name: Run Criterion.rs benchmark example - github.com/benchmark-action/github-action-benchmark-results + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - run: rustup toolchain update nightly && rustup default nightly + - name: Run benchmark + run: cd examples/criterion-rs && cargo +nightly bench -- --output-format bencher | tee output.txt + + - uses: actions/checkout@v4 + with: + repository: benchmark-action/github-action-benchmark-results + ref: 'gh-pages' + path: 'dist/other-repo' + - name: Save previous data.js + run: | + cp ./dist/other-repo/dev/bench/data.js before_data.js + + - name: Store benchmark result + uses: benchmark-action/github-action-benchmark@v1 + with: + name: Criterion.rs Benchmark + tool: 'cargo' + output-file-path: examples/criterion-rs/output.txt + fail-on-alert: true + gh-repository: 'github.com/benchmark-action/github-action-benchmark-results' + github-token: ${{ secrets.GITHUB_TOKEN }} + - run: node ./dist/scripts/ci_validate_modification.js before_data.js 'Rust Benchmark' './benchmark-data-repository' + + only-alert-with-cache: + name: Run alert check with actions/cache + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version: 20 + cache: 'npm' + - run: npm ci + - run: npm run build + - uses: actions/setup-go@v4 + with: + go-version: "stable" + - name: Run benchmark + run: cd examples/go && go test -bench 'BenchmarkFib' | tee output.txt + - name: Download previous benchmark data + uses: actions/cache@v4 + with: + path: ./cache + key: ${{ runner.os }}-ci-cache-test + - name: Store benchmark result + uses: ./ + with: + name: Alert setup example with cache + tool: 'go' + output-file-path: examples/go/output.txt + external-data-json-path: ./cache/benchmark-data.json + github-token: ${{ secrets.GITHUB_TOKEN }} + alert-threshold: '200%' + comment-on-alert: true + fail-on-alert: true + alert-comment-cc-users: '@ktrz' diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 35a258560..dbbf7102e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -8,6 +8,286 @@ on: - master jobs: + benchmarkdotnet-framework: + name: Run Benchmark.Net .Net Benchmark Framework example + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version: 20 + cache: 'npm' + - uses: actions/setup-dotnet@v3 + with: + dotnet-version: '6.0.101' # SDK Version to use. keep in line with examples/benchmarkdotnet/global.json + - run: npm ci + - run: npm run build + - name: Run benchmark + run: cd examples/benchmarkdotnet && dotnet run --exporters json --filter '*' + - name: Save previous data.js + run: | + git fetch origin gh-pages + git checkout gh-pages + cp ./dev/bench/data.js before_data.js + git checkout - + - name: Store benchmark result + uses: ./ + with: + name: Benchmark.Net Benchmark + tool: 'benchmarkdotnet' + output-file-path: examples/benchmarkdotnet/BenchmarkDotNet.Artifacts/results/Sample.Benchmarks-report-full-compressed.json + skip-fetch-gh-pages: true + fail-on-alert: true + summary-always: true + - run: node ./dist/scripts/ci_validate_modification.js before_data.js 'Benchmark.Net Benchmark' + + benchmarkjs: + name: Run JavaScript benchmark example + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version: 20 + cache: 'npm' + - run: npm ci + - run: npm run build + - name: Run benchmark + run: cd examples/benchmarkjs && npm install && node bench.js | tee output.txt + - name: Save previous data.js + run: | + git fetch origin gh-pages + git checkout gh-pages + cp ./dev/bench/data.js before_data.js + git checkout - + - name: Store benchmark result + uses: ./ + with: + name: Benchmark.js Benchmark + tool: 'benchmarkjs' + output-file-path: examples/benchmarkjs/output.txt + skip-fetch-gh-pages: true + fail-on-alert: true + summary-always: true + - run: node ./dist/scripts/ci_validate_modification.js before_data.js 'Benchmark.js Benchmark' + + catch2-framework: + name: Run Catch2 C++ Benchmark Framework example + runs-on: ubuntu-20.04 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version: 20 + cache: 'npm' + - run: npm ci + - run: npm run build + - name: Run benchmark + run: | + cd examples/catch2 + mkdir build && cd build + cmake -DCMAKE_BUILD_TYPE=Release .. + cmake --build . --config Release + ./Catch2_bench > ../benchmark_result.txt + - name: Save previous data.js + run: | + git fetch origin gh-pages + git checkout gh-pages + cp ./dev/bench/data.js before_data.js + git checkout - + - name: Store benchmark result + uses: ./ + with: + name: Catch2 Benchmark + tool: 'catch2' + output-file-path: examples/catch2/benchmark_result.txt + skip-fetch-gh-pages: true + fail-on-alert: true + summary-always: true + - run: node ./dist/scripts/ci_validate_modification.js before_data.js 'Catch2 Benchmark' + + cpp-framework: + name: Run Google C++ Benchmark Framework example + runs-on: ubuntu-20.04 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version: 20 + cache: 'npm' + - run: npm ci + - run: npm run build + - name: Cache Benchmark library + uses: actions/cache@v4 + with: + path: examples/cpp/benchmark + key: ${{ runner.os }}-googlebenchmark-v1.5.0 + - name: Run benchmark + run: | + cd examples/cpp + make json + - name: Save previous data.js + run: | + git fetch origin gh-pages + git checkout gh-pages + cp ./dev/bench/data.js before_data.js + git checkout - + - name: Store benchmark result + uses: ./ + with: + name: C++ Benchmark + tool: 'googlecpp' + output-file-path: examples/cpp/benchmark_result.json + skip-fetch-gh-pages: true + fail-on-alert: true + summary-always: true + - run: node ./dist/scripts/ci_validate_modification.js before_data.js 'C++ Benchmark' + + go: + name: Run Go benchmark example + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version: 20 + cache: 'npm' + - uses: actions/setup-go@v4 + with: + go-version: "stable" + - run: npm ci + - run: npm run build + - name: Run benchmark + run: cd examples/go && go test -bench 'BenchmarkFib' | tee output.txt + - name: Save previous data.js + run: | + git fetch origin gh-pages + git checkout gh-pages + cp ./dev/bench/data.js before_data.js + git checkout - + - name: Store benchmark result + uses: ./ + with: + name: Go Benchmark + tool: 'go' + output-file-path: examples/go/output.txt + skip-fetch-gh-pages: true + fail-on-alert: true + summary-always: true + - run: node ./dist/scripts/ci_validate_modification.js before_data.js 'Go Benchmark' + + java-jmh: + name: Run JMH Java Benchmark Framework example + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version: 20 + cache: 'npm' + - uses: actions/setup-java@v4 + with: + distribution: 'adopt' + java-version: '11' + - run: npm ci + - run: npm run build + - name: Run benchmark + run: | + cd examples/java + mvn clean verify + java -jar target/benchmarks.jar -wi 1 -i 3 -f 1 -rf json + - name: Save previous data.js + run: | + git fetch origin gh-pages + git checkout gh-pages + cp ./dev/bench/data.js before_data.js + git checkout - + - name: Store benchmark result + uses: ./ + with: + name: JMH Benchmark + tool: "jmh" + output-file-path: examples/java/jmh-result.json + skip-fetch-gh-pages: true + fail-on-alert: true + summary-always: true + - run: node ./dist/scripts/ci_validate_modification.js before_data.js 'JMH Benchmark' + + julia-benchmark: + name: Run Julia benchmark example + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version: 20 + cache: 'npm' + - run: npm ci + - run: npm run build + - uses: julia-actions/setup-julia@v1 + with: + version: '1' + arch: x64 + - name: Run benchmark + run: | + cd examples/julia + julia --project --color=yes -e ' + using Pkg; + Pkg.instantiate(); + include("fib.jl")' + - name: Save previous data.js + run: | + git fetch origin gh-pages + git checkout gh-pages + cp ./dev/bench/data.js before_data.js + git checkout - + - name: Store benchmark result + uses: ./ + with: + name: Julia benchmark + tool: 'julia' + output-file-path: examples/julia/output.json + skip-fetch-gh-pages: true + fail-on-alert: true + summary-always: true + - run: node ./dist/scripts/ci_validate_modification.js before_data.js 'Julia benchmark' + + pytest-benchmark: + name: Run Pytest benchmark example + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version: 20 + cache: 'npm' + - uses: actions/setup-python@v5 + with: + python-version: 3.9 + - run: npm ci + - run: npm run build + - name: Run benchmark + run: | + cd examples/pytest + pip install -r requirements.txt + pytest bench.py --benchmark-json output.json + - name: Save previous data.js + run: | + git fetch origin gh-pages + git checkout gh-pages + cp ./dev/bench/data.js before_data.js + git checkout - + - name: Store benchmark result + uses: ./ + with: + name: Python Benchmark with pytest-benchmark + tool: 'pytest' + output-file-path: examples/pytest/output.json + skip-fetch-gh-pages: true + fail-on-alert: true + summary-always: true + - run: node ./dist/scripts/ci_validate_modification.js before_data.js 'Python Benchmark with pytest-benchmark' + rust: name: Run Rust benchmark example runs-on: ubuntu-latest @@ -38,345 +318,89 @@ jobs: summary-always: true - run: node ./dist/scripts/ci_validate_modification.js before_data.js 'Rust Benchmark' -# go: -# name: Run Go benchmark example -# runs-on: ubuntu-latest -# steps: -# - uses: actions/checkout@v4 -# - uses: actions/setup-node@v4 -# with: -# node-version: 20 -# cache: 'npm' -# - uses: actions/setup-go@v4 -# with: -# go-version: "stable" -# - run: npm ci -# - run: npm run build -# - name: Run benchmark -# run: cd examples/go && go test -bench 'BenchmarkFib' | tee output.txt -# - name: Save previous data.js -# run: | -# git fetch origin gh-pages -# git checkout gh-pages -# cp ./dev/bench/data.js before_data.js -# git checkout - -# - name: Store benchmark result -# uses: ./ -# with: -# name: Go Benchmark -# tool: 'go' -# output-file-path: examples/go/output.txt -# skip-fetch-gh-pages: true -# fail-on-alert: true -# summary-always: true -# - run: node ./dist/scripts/ci_validate_modification.js before_data.js 'Go Benchmark' -# -# benchmarkjs: -# name: Run JavaScript benchmark example -# runs-on: ubuntu-latest -# steps: -# - uses: actions/checkout@v4 -# - uses: actions/setup-node@v4 -# with: -# node-version: 20 -# cache: 'npm' -# - run: npm ci -# - run: npm run build -# - name: Run benchmark -# run: cd examples/benchmarkjs && npm install && node bench.js | tee output.txt -# - name: Save previous data.js -# run: | -# git fetch origin gh-pages -# git checkout gh-pages -# cp ./dev/bench/data.js before_data.js -# git checkout - -# - name: Store benchmark result -# uses: ./ -# with: -# name: Benchmark.js Benchmark -# tool: 'benchmarkjs' -# output-file-path: examples/benchmarkjs/output.txt -# skip-fetch-gh-pages: true -# fail-on-alert: true -# summary-always: true -# - run: node ./dist/scripts/ci_validate_modification.js before_data.js 'Benchmark.js Benchmark' -# -# pytest-benchmark: -# name: Run Pytest benchmark example -# runs-on: ubuntu-latest -# steps: -# - uses: actions/checkout@v4 -# - uses: actions/setup-node@v4 -# with: -# node-version: 20 -# cache: 'npm' -# - uses: actions/setup-python@v5 -# with: -# python-version: 3.9 -# - run: npm ci -# - run: npm run build -# - name: Run benchmark -# run: | -# cd examples/pytest -# pip install -r requirements.txt -# pytest bench.py --benchmark-json output.json -# - name: Save previous data.js -# run: | -# git fetch origin gh-pages -# git checkout gh-pages -# cp ./dev/bench/data.js before_data.js -# git checkout - -# - name: Store benchmark result -# uses: ./ -# with: -# name: Python Benchmark with pytest-benchmark -# tool: 'pytest' -# output-file-path: examples/pytest/output.json -# skip-fetch-gh-pages: true -# fail-on-alert: true -# summary-always: true -# - run: node ./dist/scripts/ci_validate_modification.js before_data.js 'Python Benchmark with pytest-benchmark' -# -# google-benchmark-framework: -# name: Run Google C++ Benchmark Framework example -# runs-on: ubuntu-20.04 -# steps: -# - uses: actions/checkout@v4 -# - uses: actions/setup-node@v4 -# with: -# node-version: 20 -# cache: 'npm' -# - run: npm ci -# - run: npm run build -# - name: Cache Benchmark library -# uses: actions/cache@v4 -# with: -# path: examples/cpp/benchmark -# key: ${{ runner.os }}-googlebenchmark-v1.5.0 -# - name: Run benchmark -# run: | -# cd examples/cpp -# make json -# - name: Save previous data.js -# run: | -# git fetch origin gh-pages -# git checkout gh-pages -# cp ./dev/bench/data.js before_data.js -# git checkout - -# - name: Store benchmark result -# uses: ./ -# with: -# name: C++ Benchmark -# tool: 'googlecpp' -# output-file-path: examples/cpp/benchmark_result.json -# skip-fetch-gh-pages: true -# fail-on-alert: true -# summary-always: true -# - run: node ./dist/scripts/ci_validate_modification.js before_data.js 'C++ Benchmark' -# -# catch2-framework: -# name: Run Catch2 C++ Benchmark Framework example -# runs-on: ubuntu-20.04 -# steps: -# - uses: actions/checkout@v4 -# - uses: actions/setup-node@v4 -# with: -# node-version: 20 -# cache: 'npm' -# - run: npm ci -# - run: npm run build -# - name: Run benchmark -# run: | -# cd examples/catch2 -# mkdir build && cd build -# cmake -DCMAKE_BUILD_TYPE=Release .. -# cmake --build . --config Release -# ./Catch2_bench > ../benchmark_result.txt -# - name: Save previous data.js -# run: | -# git fetch origin gh-pages -# git checkout gh-pages -# cp ./dev/bench/data.js before_data.js -# git checkout - -# - name: Store benchmark result -# uses: ./ -# with: -# name: Catch2 Benchmark -# tool: 'catch2' -# output-file-path: examples/catch2/benchmark_result.txt -# skip-fetch-gh-pages: true -# fail-on-alert: true -# summary-always: true -# - run: node ./dist/scripts/ci_validate_modification.js before_data.js 'Catch2 Benchmark' -# -# julia-benchmark: -# name: Run Julia benchmark example -# runs-on: ubuntu-latest -# steps: -# - uses: actions/checkout@v4 -# - uses: actions/setup-node@v4 -# with: -# node-version: 20 -# cache: 'npm' -# - run: npm ci -# - run: npm run build -# - uses: julia-actions/setup-julia@v1 -# with: -# version: '1' -# arch: x64 -# - name: Run benchmark -# run: | -# cd examples/julia -# julia --project --color=yes -e ' -# using Pkg; -# Pkg.instantiate(); -# include("fib.jl")' -# - name: Save previous data.js -# run: | -# git fetch origin gh-pages -# git checkout gh-pages -# cp ./dev/bench/data.js before_data.js -# git checkout - -# - name: Store benchmark result -# uses: ./ -# with: -# name: Julia benchmark -# tool: 'julia' -# output-file-path: examples/julia/output.json -# skip-fetch-gh-pages: true -# fail-on-alert: true -# summary-always: true -# - run: node ./dist/scripts/ci_validate_modification.js before_data.js 'Julia benchmark' -# -# benchmarkdotnet-framework: -# name: Run Benchmark.Net .Net Benchmark Framework example -# runs-on: ubuntu-latest -# steps: -# - uses: actions/checkout@v4 -# - uses: actions/setup-node@v4 -# with: -# node-version: 20 -# cache: 'npm' -# - uses: actions/setup-dotnet@v3 -# with: -# dotnet-version: '6.0.101' # SDK Version to use. keep in line with examples/benchmarkdotnet/global.json -# - run: npm ci -# - run: npm run build -# - name: Run benchmark -# run: cd examples/benchmarkdotnet && dotnet run --exporters json --filter '*' -# - name: Save previous data.js -# run: | -# git fetch origin gh-pages -# git checkout gh-pages -# cp ./dev/bench/data.js before_data.js -# git checkout - -# - name: Store benchmark result -# uses: ./ -# with: -# name: Benchmark.Net Benchmark -# tool: 'benchmarkdotnet' -# output-file-path: examples/benchmarkdotnet/BenchmarkDotNet.Artifacts/results/Sample.Benchmarks-report-full-compressed.json -# skip-fetch-gh-pages: true -# fail-on-alert: true -# summary-always: true -# - run: node ./dist/scripts/ci_validate_modification.js before_data.js 'Benchmark.Net Benchmark' -# -# jmh: -# name: Run JMH Java Benchmark Framework example -# runs-on: ubuntu-latest -# steps: -# - uses: actions/checkout@v4 -# - uses: actions/setup-node@v4 -# with: -# node-version: 20 -# cache: 'npm' -# - uses: actions/setup-java@v4 -# with: -# distribution: 'adopt' -# java-version: '11' -# - run: npm ci -# - run: npm run build -# - name: Run benchmark -# run: | -# cd examples/java -# mvn clean verify -# java -jar target/benchmarks.jar -wi 1 -i 3 -f 1 -rf json -# - name: Save previous data.js -# run: | -# git fetch origin gh-pages -# git checkout gh-pages -# cp ./dev/bench/data.js before_data.js -# git checkout - -# - name: Store benchmark result -# uses: ./ -# with: -# name: JMH Benchmark -# tool: "jmh" -# output-file-path: examples/java/jmh-result.json -# skip-fetch-gh-pages: true -# fail-on-alert: true -# summary-always: true -# - run: node ./dist/scripts/ci_validate_modification.js before_data.js 'JMH Benchmark' -# -# only-alert-with-cache: -# name: Run alert check with actions/cache -# runs-on: ubuntu-latest -# steps: -# - uses: actions/checkout@v4 -# - uses: actions/setup-node@v4 -# with: -# node-version: 20 -# cache: 'npm' -# - run: npm ci -# - run: npm run build -# - uses: actions/setup-go@v4 -# with: -# go-version: "stable" -# - name: Run benchmark -# run: cd examples/go && go test -bench 'BenchmarkFib' | tee output.txt -# - name: Download previous benchmark data -# uses: actions/cache@v4 -# with: -# path: ./cache -# key: ${{ runner.os }}-ci-cache-test -# - name: Store benchmark result -# uses: ./ -# with: -# name: Alert setup example with cache -# tool: 'go' -# output-file-path: examples/go/output.txt -# external-data-json-path: ./cache/benchmark-data.json -# github-token: ${{ secrets.GITHUB_TOKEN }} -# alert-threshold: '200%' -# comment-on-alert: true -# fail-on-alert: true -# alert-comment-cc-users: '@ktrz' -# unit-tests: -# name: Run unit tests -# runs-on: ubuntu-latest -# steps: -# - uses: actions/checkout@v4 -# - uses: actions/setup-node@v4 -# with: -# node-version: 20 -# cache: 'npm' -# - run: npm ci -# - run: npm run coverage -# - name: Apply yamllint -# run: | -# sudo pip install yamllint -# yamllint --strict .github/workflows -# - uses: codecov/codecov-action@v2 -# -# lint: -# name: Run linting and formatting check -# runs-on: ubuntu-latest -# steps: -# - uses: actions/checkout@v4 -# - uses: actions/setup-node@v4 -# with: -# node-version: 20 -# cache: 'npm' -# - run: npm ci -# - run: npm run lint -# - run: npm run format:check + rust-criterion-rs-framework: + name: Run Criterion.rs benchmark example + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - run: rustup toolchain update nightly && rustup default nightly + - name: Run benchmark + run: cd examples/criterion-rs && cargo +nightly bench -- --output-format bencher | tee output.txt + - name: Save previous data.js + run: | + git fetch origin gh-pages + git checkout gh-pages + cp ./dev/bench/data.js before_data.js + git checkout - + - name: Store benchmark result + uses: benchmark-action/github-action-benchmark@v1 + with: + name: Criterion.rs Benchmark + tool: 'cargo' + output-file-path: examples/criterion-rs/output.txt + fail-on-alert: true + summary-always: true + - run: node ./dist/scripts/ci_validate_modification.js before_data.js 'Rust Benchmark' + + only-alert-with-cache: + name: Run alert check with actions/cache + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version: 20 + cache: 'npm' + - run: npm ci + - run: npm run build + - uses: actions/setup-go@v4 + with: + go-version: "stable" + - name: Run benchmark + run: cd examples/go && go test -bench 'BenchmarkFib' | tee output.txt + - name: Download previous benchmark data + uses: actions/cache@v4 + with: + path: ./cache + key: ${{ runner.os }}-ci-cache-test + - name: Store benchmark result + uses: ./ + with: + name: Alert setup example with cache + tool: 'go' + output-file-path: examples/go/output.txt + external-data-json-path: ./cache/benchmark-data.json + github-token: ${{ secrets.GITHUB_TOKEN }} + alert-threshold: '200%' + comment-on-alert: true + fail-on-alert: true + alert-comment-cc-users: '@ktrz' + unit-tests: + name: Run unit tests + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version: 20 + cache: 'npm' + - run: npm ci + - run: npm run coverage + - name: Apply yamllint + run: | + sudo pip install yamllint + yamllint --strict .github/workflows + - uses: codecov/codecov-action@v2 + + lint: + name: Run linting and formatting check + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version: 20 + cache: 'npm' + - run: npm ci + - run: npm run lint + - run: npm run format:check diff --git a/.github/workflows/cpp.yml b/.github/workflows/cpp.yml index 010ebb6f5..1034c0eb7 100644 --- a/.github/workflows/cpp.yml +++ b/.github/workflows/cpp.yml @@ -10,7 +10,7 @@ permissions: jobs: benchmark: - name: Run C++ benchmark example + name: Run Google C++ Benchmark Framework example runs-on: ubuntu-20.04 steps: - uses: actions/checkout@v4 diff --git a/.github/workflows/criterion-rs.yml b/.github/workflows/rust-criterion-rs.yml similarity index 95% rename from .github/workflows/criterion-rs.yml rename to .github/workflows/rust-criterion-rs.yml index 98231cc5e..4034b9d3b 100644 --- a/.github/workflows/criterion-rs.yml +++ b/.github/workflows/rust-criterion-rs.yml @@ -21,7 +21,7 @@ jobs: - name: Store benchmark result uses: benchmark-action/github-action-benchmark@v1 with: - name: Rust Benchmark + name: Criterion.rs Benchmark tool: 'cargo' output-file-path: examples/criterion-rs/output.txt github-token: ${{ secrets.GITHUB_TOKEN }} @@ -35,7 +35,7 @@ jobs: - name: Store benchmark result - separate results repo uses: benchmark-action/github-action-benchmark@v1 with: - name: Rust Benchmark + name: Criterion.rs Benchmark tool: 'cargo' output-file-path: examples/criterion-rs/output.txt github-token: ${{ secrets.BENCHMARK_ACTION_BOT_TOKEN }} From fd4d758352d623e3f6e48d6133ba53ec20bbca5c Mon Sep 17 00:00:00 2001 From: Chris Trzesniewski Date: Tue, 26 Mar 2024 10:22:09 +0100 Subject: [PATCH 3/8] cleanup --- .github/workflows/ci-results-repo.yml | 4 ++-- src/write.ts | 1 - 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci-results-repo.yml b/.github/workflows/ci-results-repo.yml index 27ec9f6f4..947de91a3 100644 --- a/.github/workflows/ci-results-repo.yml +++ b/.github/workflows/ci-results-repo.yml @@ -234,7 +234,7 @@ jobs: fail-on-alert: true gh-repository: 'github.com/benchmark-action/github-action-benchmark-results' github-token: ${{ secrets.GITHUB_TOKEN }} - - run: node ./dist/scripts/ci_validate_modification.js before_data.js 'JMH Benchmark' + - run: node ./dist/scripts/ci_validate_modification.js before_data.js 'JMH Benchmark' './benchmark-data-repository' julia-benchmark: name: Run Julia benchmark example - github.com/benchmark-action/github-action-benchmark-results @@ -381,7 +381,7 @@ jobs: fail-on-alert: true gh-repository: 'github.com/benchmark-action/github-action-benchmark-results' github-token: ${{ secrets.GITHUB_TOKEN }} - - run: node ./dist/scripts/ci_validate_modification.js before_data.js 'Rust Benchmark' './benchmark-data-repository' + - run: node ./dist/scripts/ci_validate_modification.js before_data.js 'Criterion.rs Benchmark' './benchmark-data-repository' only-alert-with-cache: name: Run alert check with actions/cache diff --git a/src/write.ts b/src/write.ts index d774454e8..3c71823a7 100644 --- a/src/write.ts +++ b/src/write.ts @@ -94,7 +94,6 @@ interface Alert { function findAlerts(curSuite: Benchmark, prevSuite: Benchmark, threshold: number): Alert[] { core.debug(`Comparing current:${curSuite.commit.id} and prev:${prevSuite.commit.id} for alert`); - core.debug(`Previous suite benches: ${JSON.stringify(prevSuite.benches)}`); const alerts = []; for (const current of curSuite.benches) { From e05a541b4767a68f0294063757437c42b0a6b371 Mon Sep 17 00:00:00 2001 From: Chris Trzesniewski Date: Tue, 26 Mar 2024 10:27:56 +0100 Subject: [PATCH 4/8] build for criterion --- .github/workflows/ci-results-repo.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/ci-results-repo.yml b/.github/workflows/ci-results-repo.yml index 947de91a3..fbdd99f78 100644 --- a/.github/workflows/ci-results-repo.yml +++ b/.github/workflows/ci-results-repo.yml @@ -359,6 +359,12 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version: 20 + cache: 'npm' + - run: npm ci + - run: npm run build - run: rustup toolchain update nightly && rustup default nightly - name: Run benchmark run: cd examples/criterion-rs && cargo +nightly bench -- --output-format bencher | tee output.txt From 6aaea4ecf488300c2fb9c18c5098d6d5e2c4f4ad Mon Sep 17 00:00:00 2001 From: Chris Trzesniewski Date: Tue, 26 Mar 2024 10:32:17 +0100 Subject: [PATCH 5/8] fix java ci run --- .github/workflows/ci-results-repo.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/ci-results-repo.yml b/.github/workflows/ci-results-repo.yml index fbdd99f78..933a52e39 100644 --- a/.github/workflows/ci-results-repo.yml +++ b/.github/workflows/ci-results-repo.yml @@ -230,7 +230,6 @@ jobs: name: JMH Benchmark tool: "jmh" output-file-path: examples/java/jmh-result.json - skip-fetch-gh-pages: true fail-on-alert: true gh-repository: 'github.com/benchmark-action/github-action-benchmark-results' github-token: ${{ secrets.GITHUB_TOKEN }} From 3ad139122b1b93b4e8c1c004529f0f4cb6fd2cad Mon Sep 17 00:00:00 2001 From: Chris Trzesniewski Date: Tue, 26 Mar 2024 10:39:54 +0100 Subject: [PATCH 6/8] fix criterion rs run --- .github/workflows/ci.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index dbbf7102e..1fb6b0dfd 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -323,6 +323,12 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version: 20 + cache: 'npm' + - run: npm ci + - run: npm run build - run: rustup toolchain update nightly && rustup default nightly - name: Run benchmark run: cd examples/criterion-rs && cargo +nightly bench -- --output-format bencher | tee output.txt From 338b0bf7be9002fcf81b12a12181278840c4dca2 Mon Sep 17 00:00:00 2001 From: Chris Trzesniewski Date: Tue, 26 Mar 2024 11:11:52 +0100 Subject: [PATCH 7/8] fix criterion rs run --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1fb6b0dfd..993ac985c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -346,7 +346,7 @@ jobs: output-file-path: examples/criterion-rs/output.txt fail-on-alert: true summary-always: true - - run: node ./dist/scripts/ci_validate_modification.js before_data.js 'Rust Benchmark' + - run: node ./dist/scripts/ci_validate_modification.js before_data.js 'Criterion.rs Benchmark' only-alert-with-cache: name: Run alert check with actions/cache From 0402ad9cc34cb24b646bdb6b21e4e519060d24f6 Mon Sep 17 00:00:00 2001 From: Chris Trzesniewski Date: Tue, 26 Mar 2024 11:31:21 +0100 Subject: [PATCH 8/8] add changelog entry --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index d4b5818c9..b6b3225c5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,5 @@ ## Unreleased +- **fix** Rust benchmarks not comparing to baseline (#235) # [v1.19.3](https://github.com/benchmark-action/github-action-benchmark/releases/tag/v1.19.3) - 02 Feb 2024