From a6c4f1c31d68159af7621fe2f45c3c33b6e5fb95 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Chris=20Trze=C5=9Bniewski?= Date: Tue, 26 Mar 2024 11:49:09 +0100 Subject: [PATCH] fix: 231 rust bench not comparing to baseline (#235) * sort jobs in ci workflows * add missing benches to ci jobs * rename CriterionRS bench to distinguish between default Rust and Criterion benchmarks --- .github/workflows/ci-results-repo.yml | 228 ++++++++++++------ .github/workflows/ci.yml | 208 +++++++++------- .github/workflows/cpp.yml | 2 +- ...criterion-rs.yml => rust-criterion-rs.yml} | 4 +- .gitignore | 1 + CHANGELOG.md | 1 + 6 files changed, 275 insertions(+), 169 deletions(-) rename .github/workflows/{criterion-rs.yml => rust-criterion-rs.yml} (95%) diff --git a/.github/workflows/ci-results-repo.yml b/.github/workflows/ci-results-repo.yml index 7358f7d8a..933a52e39 100644 --- a/.github/workflows/ci-results-repo.yml +++ b/.github/workflows/ci-results-repo.yml @@ -8,20 +8,22 @@ on: - master jobs: - rust: - name: Run Rust benchmark example - github.com/benchmark-action/github-action-benchmark-results + benchmarkdotnet-framework: + name: Run Benchmark.Net .Net Benchmark Framework example - github.com/benchmark-action/github-action-benchmark-results runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 + - uses: actions/setup-dotnet@v3 + with: + dotnet-version: '6.0.101' # SDK Version to use. keep in line with examples/benchmarkdotnet/global.json - uses: actions/setup-node@v4 with: node-version: 20 cache: 'npm' - run: npm ci - run: npm run build - - run: rustup toolchain update nightly && rustup default nightly - name: Run benchmark - run: cd examples/rust && cargo +nightly bench | tee output.txt + run: cd examples/benchmarkdotnet && dotnet run --exporters json --filter '*' - uses: actions/checkout@v4 with: @@ -35,16 +37,16 @@ jobs: - name: Store benchmark result uses: ./ with: - name: Rust Benchmark - tool: 'cargo' - output-file-path: examples/rust/output.txt + name: Benchmark.Net Benchmark + tool: 'benchmarkdotnet' + output-file-path: examples/benchmarkdotnet/BenchmarkDotNet.Artifacts/results/Sample.Benchmarks-report-full-compressed.json fail-on-alert: true gh-repository: 'github.com/benchmark-action/github-action-benchmark-results' github-token: ${{ secrets.GITHUB_TOKEN }} - - run: node ./dist/scripts/ci_validate_modification.js before_data.js 'Rust Benchmark' './benchmark-data-repository' + - run: node ./dist/scripts/ci_validate_modification.js before_data.js 'Benchmark.Net Benchmark' './benchmark-data-repository' - go: - name: Run Go benchmark example - github.com/benchmark-action/github-action-benchmark-results + benchmarkjs: + name: Run JavaScript benchmark example - github.com/benchmark-action/github-action-benchmark-results runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 @@ -52,13 +54,10 @@ jobs: with: node-version: 20 cache: 'npm' - - uses: actions/setup-go@v4 - with: - go-version: "stable" - run: npm ci - run: npm run build - name: Run benchmark - run: cd examples/go && go test -bench 'BenchmarkFib' | tee output.txt + run: cd examples/benchmarkjs && npm install && node bench.js | tee output.txt - uses: actions/checkout@v4 with: @@ -72,17 +71,17 @@ jobs: - name: Store benchmark result uses: ./ with: - name: Go Benchmark - tool: 'go' - output-file-path: examples/go/output.txt + name: Benchmark.js Benchmark + tool: 'benchmarkjs' + output-file-path: examples/benchmarkjs/output.txt fail-on-alert: true gh-repository: 'github.com/benchmark-action/github-action-benchmark-results' github-token: ${{ secrets.GITHUB_TOKEN }} - - run: node ./dist/scripts/ci_validate_modification.js before_data.js 'Go Benchmark' './benchmark-data-repository' + - run: node ./dist/scripts/ci_validate_modification.js before_data.js 'Benchmark.js Benchmark' './benchmark-data-repository' - benchmarkjs: - name: Run JavaScript benchmark example - github.com/benchmark-action/github-action-benchmark-results - runs-on: ubuntu-latest + catch2-framework: + name: Run Catch2 C++ Benchmark Framework example - github.com/benchmark-action/github-action-benchmark-results + runs-on: ubuntu-20.04 steps: - uses: actions/checkout@v4 - uses: actions/setup-node@v4 @@ -92,7 +91,12 @@ jobs: - run: npm ci - run: npm run build - name: Run benchmark - run: cd examples/benchmarkjs && npm install && node bench.js | tee output.txt + run: | + cd examples/catch2 + mkdir build && cd build + cmake -DCMAKE_BUILD_TYPE=Release .. + cmake --build . --config Release + ./Catch2_bench > ../benchmark_result.txt - uses: actions/checkout@v4 with: @@ -106,33 +110,34 @@ jobs: - name: Store benchmark result uses: ./ with: - name: Benchmark.js Benchmark - tool: 'benchmarkjs' - output-file-path: examples/benchmarkjs/output.txt + name: Catch2 Benchmark + tool: 'catch2' + output-file-path: examples/catch2/benchmark_result.txt fail-on-alert: true gh-repository: 'github.com/benchmark-action/github-action-benchmark-results' github-token: ${{ secrets.GITHUB_TOKEN }} - - run: node ./dist/scripts/ci_validate_modification.js before_data.js 'Benchmark.js Benchmark' './benchmark-data-repository' + - run: node ./dist/scripts/ci_validate_modification.js before_data.js 'Catch2 Benchmark' './benchmark-data-repository' - pytest-benchmark: - name: Run Pytest benchmark example - github.com/benchmark-action/github-action-benchmark-results - runs-on: ubuntu-latest + cpp-framework: + name: Run Google C++ Benchmark Framework example - github.com/benchmark-action/github-action-benchmark-results + runs-on: ubuntu-20.04 steps: - uses: actions/checkout@v4 - uses: actions/setup-node@v4 with: node-version: 20 cache: 'npm' - - uses: actions/setup-python@v5 - with: - python-version: 3.9 - run: npm ci - run: npm run build + - name: Cache Benchmark library + uses: actions/cache@v4 + with: + path: examples/cpp/benchmark + key: ${{ runner.os }}-googlebenchmark-v1.5.0 - name: Run benchmark run: | - cd examples/pytest - pip install -r requirements.txt - pytest bench.py --benchmark-json output.json + cd examples/cpp + make json - uses: actions/checkout@v4 with: @@ -146,34 +151,30 @@ jobs: - name: Store benchmark result uses: ./ with: - name: Python Benchmark with pytest-benchmark - tool: 'pytest' - output-file-path: examples/pytest/output.json + name: C++ Benchmark + tool: 'googlecpp' + output-file-path: examples/cpp/benchmark_result.json fail-on-alert: true gh-repository: 'github.com/benchmark-action/github-action-benchmark-results' github-token: ${{ secrets.GITHUB_TOKEN }} - - run: node ./dist/scripts/ci_validate_modification.js before_data.js 'Python Benchmark with pytest-benchmark' './benchmark-data-repository' + - run: node ./dist/scripts/ci_validate_modification.js before_data.js 'C++ Benchmark' './benchmark-data-repository' - google-benchmark-framework: - name: Run Google C++ Benchmark Framework example - github.com/benchmark-action/github-action-benchmark-results - runs-on: ubuntu-20.04 + go: + name: Run Go benchmark example - github.com/benchmark-action/github-action-benchmark-results + runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - uses: actions/setup-node@v4 with: node-version: 20 cache: 'npm' + - uses: actions/setup-go@v4 + with: + go-version: "stable" - run: npm ci - run: npm run build - - name: Cache Benchmark library - uses: actions/cache@v4 - with: - path: examples/cpp/benchmark - key: ${{ runner.os }}-googlebenchmark-v1.5.0 - name: Run benchmark - run: | - cd examples/cpp - make json + run: cd examples/go && go test -bench 'BenchmarkFib' | tee output.txt - uses: actions/checkout@v4 with: @@ -187,33 +188,34 @@ jobs: - name: Store benchmark result uses: ./ with: - name: C++ Benchmark - tool: 'googlecpp' - output-file-path: examples/cpp/benchmark_result.json + name: Go Benchmark + tool: 'go' + output-file-path: examples/go/output.txt fail-on-alert: true gh-repository: 'github.com/benchmark-action/github-action-benchmark-results' github-token: ${{ secrets.GITHUB_TOKEN }} - - run: node ./dist/scripts/ci_validate_modification.js before_data.js 'C++ Benchmark' './benchmark-data-repository' + - run: node ./dist/scripts/ci_validate_modification.js before_data.js 'Go Benchmark' './benchmark-data-repository' - catch2-framework: - name: Run Catch2 C++ Benchmark Framework example - github.com/benchmark-action/github-action-benchmark-results - runs-on: ubuntu-20.04 + java-jmh: + name: Run JMH Java Benchmark Framework example + runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - uses: actions/setup-node@v4 with: node-version: 20 cache: 'npm' + - uses: actions/setup-java@v4 + with: + distribution: 'adopt' + java-version: '11' - run: npm ci - run: npm run build - name: Run benchmark run: | - cd examples/catch2 - mkdir build && cd build - cmake -DCMAKE_BUILD_TYPE=Release .. - cmake --build . --config Release - ./Catch2_bench > ../benchmark_result.txt - + cd examples/java + mvn clean verify + java -jar target/benchmarks.jar -wi 1 -i 3 -f 1 -rf json - uses: actions/checkout@v4 with: repository: benchmark-action/github-action-benchmark-results @@ -222,17 +224,16 @@ jobs: - name: Save previous data.js run: | cp ./dist/other-repo/dev/bench/data.js before_data.js - - name: Store benchmark result uses: ./ with: - name: Catch2 Benchmark - tool: 'catch2' - output-file-path: examples/catch2/benchmark_result.txt + name: JMH Benchmark + tool: "jmh" + output-file-path: examples/java/jmh-result.json fail-on-alert: true gh-repository: 'github.com/benchmark-action/github-action-benchmark-results' github-token: ${{ secrets.GITHUB_TOKEN }} - - run: node ./dist/scripts/ci_validate_modification.js before_data.js 'Catch2 Benchmark' './benchmark-data-repository' + - run: node ./dist/scripts/ci_validate_modification.js before_data.js 'JMH Benchmark' './benchmark-data-repository' julia-benchmark: name: Run Julia benchmark example - github.com/benchmark-action/github-action-benchmark-results @@ -277,22 +278,60 @@ jobs: github-token: ${{ secrets.GITHUB_TOKEN }} - run: node ./dist/scripts/ci_validate_modification.js before_data.js 'Julia benchmark' './benchmark-data-repository' - benchmarkdotnet-framework: - name: Run Benchmark.Net .Net Benchmark Framework example - github.com/benchmark-action/github-action-benchmark-results + pytest-benchmark: + name: Run Pytest benchmark example - github.com/benchmark-action/github-action-benchmark-results runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - - uses: actions/setup-dotnet@v3 + - uses: actions/setup-node@v4 with: - dotnet-version: '6.0.101' # SDK Version to use. keep in line with examples/benchmarkdotnet/global.json + node-version: 20 + cache: 'npm' + - uses: actions/setup-python@v5 + with: + python-version: 3.9 + - run: npm ci + - run: npm run build + - name: Run benchmark + run: | + cd examples/pytest + pip install -r requirements.txt + pytest bench.py --benchmark-json output.json + + - uses: actions/checkout@v4 + with: + repository: benchmark-action/github-action-benchmark-results + ref: 'gh-pages' + path: 'dist/other-repo' + - name: Save previous data.js + run: | + cp ./dist/other-repo/dev/bench/data.js before_data.js + + - name: Store benchmark result + uses: ./ + with: + name: Python Benchmark with pytest-benchmark + tool: 'pytest' + output-file-path: examples/pytest/output.json + fail-on-alert: true + gh-repository: 'github.com/benchmark-action/github-action-benchmark-results' + github-token: ${{ secrets.GITHUB_TOKEN }} + - run: node ./dist/scripts/ci_validate_modification.js before_data.js 'Python Benchmark with pytest-benchmark' './benchmark-data-repository' + + rust: + name: Run Rust benchmark example - github.com/benchmark-action/github-action-benchmark-results + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 - uses: actions/setup-node@v4 with: node-version: 20 cache: 'npm' - run: npm ci - run: npm run build + - run: rustup toolchain update nightly && rustup default nightly - name: Run benchmark - run: cd examples/benchmarkdotnet && dotnet run --exporters json --filter '*' + run: cd examples/rust && cargo +nightly bench | tee output.txt - uses: actions/checkout@v4 with: @@ -306,13 +345,48 @@ jobs: - name: Store benchmark result uses: ./ with: - name: Benchmark.Net Benchmark - tool: 'benchmarkdotnet' - output-file-path: examples/benchmarkdotnet/BenchmarkDotNet.Artifacts/results/Sample.Benchmarks-report-full-compressed.json + name: Rust Benchmark + tool: 'cargo' + output-file-path: examples/rust/output.txt fail-on-alert: true gh-repository: 'github.com/benchmark-action/github-action-benchmark-results' github-token: ${{ secrets.GITHUB_TOKEN }} - - run: node ./dist/scripts/ci_validate_modification.js before_data.js 'Benchmark.Net Benchmark' './benchmark-data-repository' + - run: node ./dist/scripts/ci_validate_modification.js before_data.js 'Rust Benchmark' './benchmark-data-repository' + + rust-criterion-rs-framework: + name: Run Criterion.rs benchmark example - github.com/benchmark-action/github-action-benchmark-results + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version: 20 + cache: 'npm' + - run: npm ci + - run: npm run build + - run: rustup toolchain update nightly && rustup default nightly + - name: Run benchmark + run: cd examples/criterion-rs && cargo +nightly bench -- --output-format bencher | tee output.txt + + - uses: actions/checkout@v4 + with: + repository: benchmark-action/github-action-benchmark-results + ref: 'gh-pages' + path: 'dist/other-repo' + - name: Save previous data.js + run: | + cp ./dist/other-repo/dev/bench/data.js before_data.js + + - name: Store benchmark result + uses: benchmark-action/github-action-benchmark@v1 + with: + name: Criterion.rs Benchmark + tool: 'cargo' + output-file-path: examples/criterion-rs/output.txt + fail-on-alert: true + gh-repository: 'github.com/benchmark-action/github-action-benchmark-results' + github-token: ${{ secrets.GITHUB_TOKEN }} + - run: node ./dist/scripts/ci_validate_modification.js before_data.js 'Criterion.rs Benchmark' './benchmark-data-repository' only-alert-with-cache: name: Run alert check with actions/cache diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index eed878cb8..993ac985c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -8,38 +8,8 @@ on: - master jobs: - rust: - name: Run Rust benchmark example - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-node@v4 - with: - node-version: 20 - cache: 'npm' - - run: npm ci - - run: npm run build - - run: rustup toolchain update nightly && rustup default nightly - - name: Run benchmark - run: cd examples/rust && cargo +nightly bench | tee output.txt - - name: Save previous data.js - run: | - git fetch origin gh-pages - git checkout gh-pages - cp ./dev/bench/data.js before_data.js - git checkout - - - name: Store benchmark result - uses: ./ - with: - name: Rust Benchmark - tool: 'cargo' - output-file-path: examples/rust/output.txt - fail-on-alert: true - summary-always: true - - run: node ./dist/scripts/ci_validate_modification.js before_data.js 'Rust Benchmark' - - go: - name: Run Go benchmark example + benchmarkdotnet-framework: + name: Run Benchmark.Net .Net Benchmark Framework example runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 @@ -47,13 +17,13 @@ jobs: with: node-version: 20 cache: 'npm' - - uses: actions/setup-go@v4 + - uses: actions/setup-dotnet@v3 with: - go-version: "stable" + dotnet-version: '6.0.101' # SDK Version to use. keep in line with examples/benchmarkdotnet/global.json - run: npm ci - run: npm run build - name: Run benchmark - run: cd examples/go && go test -bench 'BenchmarkFib' | tee output.txt + run: cd examples/benchmarkdotnet && dotnet run --exporters json --filter '*' - name: Save previous data.js run: | git fetch origin gh-pages @@ -63,13 +33,13 @@ jobs: - name: Store benchmark result uses: ./ with: - name: Go Benchmark - tool: 'go' - output-file-path: examples/go/output.txt + name: Benchmark.Net Benchmark + tool: 'benchmarkdotnet' + output-file-path: examples/benchmarkdotnet/BenchmarkDotNet.Artifacts/results/Sample.Benchmarks-report-full-compressed.json skip-fetch-gh-pages: true fail-on-alert: true summary-always: true - - run: node ./dist/scripts/ci_validate_modification.js before_data.js 'Go Benchmark' + - run: node ./dist/scripts/ci_validate_modification.js before_data.js 'Benchmark.Net Benchmark' benchmarkjs: name: Run JavaScript benchmark example @@ -101,25 +71,24 @@ jobs: summary-always: true - run: node ./dist/scripts/ci_validate_modification.js before_data.js 'Benchmark.js Benchmark' - pytest-benchmark: - name: Run Pytest benchmark example - runs-on: ubuntu-latest + catch2-framework: + name: Run Catch2 C++ Benchmark Framework example + runs-on: ubuntu-20.04 steps: - uses: actions/checkout@v4 - uses: actions/setup-node@v4 with: node-version: 20 cache: 'npm' - - uses: actions/setup-python@v5 - with: - python-version: 3.9 - run: npm ci - run: npm run build - name: Run benchmark run: | - cd examples/pytest - pip install -r requirements.txt - pytest bench.py --benchmark-json output.json + cd examples/catch2 + mkdir build && cd build + cmake -DCMAKE_BUILD_TYPE=Release .. + cmake --build . --config Release + ./Catch2_bench > ../benchmark_result.txt - name: Save previous data.js run: | git fetch origin gh-pages @@ -129,15 +98,15 @@ jobs: - name: Store benchmark result uses: ./ with: - name: Python Benchmark with pytest-benchmark - tool: 'pytest' - output-file-path: examples/pytest/output.json + name: Catch2 Benchmark + tool: 'catch2' + output-file-path: examples/catch2/benchmark_result.txt skip-fetch-gh-pages: true fail-on-alert: true summary-always: true - - run: node ./dist/scripts/ci_validate_modification.js before_data.js 'Python Benchmark with pytest-benchmark' + - run: node ./dist/scripts/ci_validate_modification.js before_data.js 'Catch2 Benchmark' - google-benchmark-framework: + cpp-framework: name: Run Google C++ Benchmark Framework example runs-on: ubuntu-20.04 steps: @@ -174,24 +143,59 @@ jobs: summary-always: true - run: node ./dist/scripts/ci_validate_modification.js before_data.js 'C++ Benchmark' - catch2-framework: - name: Run Catch2 C++ Benchmark Framework example - runs-on: ubuntu-20.04 + go: + name: Run Go benchmark example + runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - uses: actions/setup-node@v4 with: node-version: 20 cache: 'npm' + - uses: actions/setup-go@v4 + with: + go-version: "stable" - run: npm ci - run: npm run build - name: Run benchmark + run: cd examples/go && go test -bench 'BenchmarkFib' | tee output.txt + - name: Save previous data.js run: | - cd examples/catch2 - mkdir build && cd build - cmake -DCMAKE_BUILD_TYPE=Release .. - cmake --build . --config Release - ./Catch2_bench > ../benchmark_result.txt + git fetch origin gh-pages + git checkout gh-pages + cp ./dev/bench/data.js before_data.js + git checkout - + - name: Store benchmark result + uses: ./ + with: + name: Go Benchmark + tool: 'go' + output-file-path: examples/go/output.txt + skip-fetch-gh-pages: true + fail-on-alert: true + summary-always: true + - run: node ./dist/scripts/ci_validate_modification.js before_data.js 'Go Benchmark' + + java-jmh: + name: Run JMH Java Benchmark Framework example + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version: 20 + cache: 'npm' + - uses: actions/setup-java@v4 + with: + distribution: 'adopt' + java-version: '11' + - run: npm ci + - run: npm run build + - name: Run benchmark + run: | + cd examples/java + mvn clean verify + java -jar target/benchmarks.jar -wi 1 -i 3 -f 1 -rf json - name: Save previous data.js run: | git fetch origin gh-pages @@ -201,13 +205,13 @@ jobs: - name: Store benchmark result uses: ./ with: - name: Catch2 Benchmark - tool: 'catch2' - output-file-path: examples/catch2/benchmark_result.txt + name: JMH Benchmark + tool: "jmh" + output-file-path: examples/java/jmh-result.json skip-fetch-gh-pages: true fail-on-alert: true summary-always: true - - run: node ./dist/scripts/ci_validate_modification.js before_data.js 'Catch2 Benchmark' + - run: node ./dist/scripts/ci_validate_modification.js before_data.js 'JMH Benchmark' julia-benchmark: name: Run Julia benchmark example @@ -248,8 +252,8 @@ jobs: summary-always: true - run: node ./dist/scripts/ci_validate_modification.js before_data.js 'Julia benchmark' - benchmarkdotnet-framework: - name: Run Benchmark.Net .Net Benchmark Framework example + pytest-benchmark: + name: Run Pytest benchmark example runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 @@ -257,13 +261,16 @@ jobs: with: node-version: 20 cache: 'npm' - - uses: actions/setup-dotnet@v3 + - uses: actions/setup-python@v5 with: - dotnet-version: '6.0.101' # SDK Version to use. keep in line with examples/benchmarkdotnet/global.json + python-version: 3.9 - run: npm ci - run: npm run build - name: Run benchmark - run: cd examples/benchmarkdotnet && dotnet run --exporters json --filter '*' + run: | + cd examples/pytest + pip install -r requirements.txt + pytest bench.py --benchmark-json output.json - name: Save previous data.js run: | git fetch origin gh-pages @@ -273,16 +280,16 @@ jobs: - name: Store benchmark result uses: ./ with: - name: Benchmark.Net Benchmark - tool: 'benchmarkdotnet' - output-file-path: examples/benchmarkdotnet/BenchmarkDotNet.Artifacts/results/Sample.Benchmarks-report-full-compressed.json + name: Python Benchmark with pytest-benchmark + tool: 'pytest' + output-file-path: examples/pytest/output.json skip-fetch-gh-pages: true fail-on-alert: true summary-always: true - - run: node ./dist/scripts/ci_validate_modification.js before_data.js 'Benchmark.Net Benchmark' + - run: node ./dist/scripts/ci_validate_modification.js before_data.js 'Python Benchmark with pytest-benchmark' - jmh: - name: Run JMH Java Benchmark Framework example + rust: + name: Run Rust benchmark example runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 @@ -290,17 +297,11 @@ jobs: with: node-version: 20 cache: 'npm' - - uses: actions/setup-java@v4 - with: - distribution: 'adopt' - java-version: '11' - run: npm ci - run: npm run build + - run: rustup toolchain update nightly && rustup default nightly - name: Run benchmark - run: | - cd examples/java - mvn clean verify - java -jar target/benchmarks.jar -wi 1 -i 3 -f 1 -rf json + run: cd examples/rust && cargo +nightly bench | tee output.txt - name: Save previous data.js run: | git fetch origin gh-pages @@ -310,13 +311,42 @@ jobs: - name: Store benchmark result uses: ./ with: - name: JMH Benchmark - tool: "jmh" - output-file-path: examples/java/jmh-result.json - skip-fetch-gh-pages: true + name: Rust Benchmark + tool: 'cargo' + output-file-path: examples/rust/output.txt fail-on-alert: true summary-always: true - - run: node ./dist/scripts/ci_validate_modification.js before_data.js 'JMH Benchmark' + - run: node ./dist/scripts/ci_validate_modification.js before_data.js 'Rust Benchmark' + + rust-criterion-rs-framework: + name: Run Criterion.rs benchmark example + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version: 20 + cache: 'npm' + - run: npm ci + - run: npm run build + - run: rustup toolchain update nightly && rustup default nightly + - name: Run benchmark + run: cd examples/criterion-rs && cargo +nightly bench -- --output-format bencher | tee output.txt + - name: Save previous data.js + run: | + git fetch origin gh-pages + git checkout gh-pages + cp ./dev/bench/data.js before_data.js + git checkout - + - name: Store benchmark result + uses: benchmark-action/github-action-benchmark@v1 + with: + name: Criterion.rs Benchmark + tool: 'cargo' + output-file-path: examples/criterion-rs/output.txt + fail-on-alert: true + summary-always: true + - run: node ./dist/scripts/ci_validate_modification.js before_data.js 'Criterion.rs Benchmark' only-alert-with-cache: name: Run alert check with actions/cache diff --git a/.github/workflows/cpp.yml b/.github/workflows/cpp.yml index 010ebb6f5..1034c0eb7 100644 --- a/.github/workflows/cpp.yml +++ b/.github/workflows/cpp.yml @@ -10,7 +10,7 @@ permissions: jobs: benchmark: - name: Run C++ benchmark example + name: Run Google C++ Benchmark Framework example runs-on: ubuntu-20.04 steps: - uses: actions/checkout@v4 diff --git a/.github/workflows/criterion-rs.yml b/.github/workflows/rust-criterion-rs.yml similarity index 95% rename from .github/workflows/criterion-rs.yml rename to .github/workflows/rust-criterion-rs.yml index 98231cc5e..4034b9d3b 100644 --- a/.github/workflows/criterion-rs.yml +++ b/.github/workflows/rust-criterion-rs.yml @@ -21,7 +21,7 @@ jobs: - name: Store benchmark result uses: benchmark-action/github-action-benchmark@v1 with: - name: Rust Benchmark + name: Criterion.rs Benchmark tool: 'cargo' output-file-path: examples/criterion-rs/output.txt github-token: ${{ secrets.GITHUB_TOKEN }} @@ -35,7 +35,7 @@ jobs: - name: Store benchmark result - separate results repo uses: benchmark-action/github-action-benchmark@v1 with: - name: Rust Benchmark + name: Criterion.rs Benchmark tool: 'cargo' output-file-path: examples/criterion-rs/output.txt github-token: ${{ secrets.BENCHMARK_ACTION_BOT_TOKEN }} diff --git a/.gitignore b/.gitignore index c6a69e261..9578ebeea 100644 --- a/.gitignore +++ b/.gitignore @@ -12,3 +12,4 @@ /.idea target/ jmh-result.json +.env diff --git a/CHANGELOG.md b/CHANGELOG.md index d4b5818c9..b6b3225c5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,5 @@ ## Unreleased +- **fix** Rust benchmarks not comparing to baseline (#235) # [v1.19.3](https://github.com/benchmark-action/github-action-benchmark/releases/tag/v1.19.3) - 02 Feb 2024