fix: 231 rust bench not comparing to baseline (#235)

* sort jobs in ci workflows * add missing benches to ci jobs * rename CriterionRS bench to distinguish between default Rust and Criterion benchmarks
benchmark-action · Mar 26, 2024 · a6c4f1c · a6c4f1c
1 parent e7180f5
commit a6c4f1c
Show file tree

Hide file tree

Showing 6 changed files with 275 additions and 169 deletions.
diff --git a/.github/workflows/ci-results-repo.yml b/.github/workflows/ci-results-repo.yml
@@ -8,20 +8,22 @@ on:
       - master
 
 jobs:
-  rust:
-    name: Run Rust benchmark example - github.com/benchmark-action/github-action-benchmark-results
+  benchmarkdotnet-framework:
+    name: Run Benchmark.Net .Net Benchmark Framework example - github.com/benchmark-action/github-action-benchmark-results
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
+      - uses: actions/setup-dotnet@v3
+        with:
+          dotnet-version: '6.0.101'  # SDK Version to use. keep in line with examples/benchmarkdotnet/global.json
       - uses: actions/setup-node@v4
         with:
           node-version: 20
           cache: 'npm'
       - run: npm ci
       - run: npm run build
-      - run: rustup toolchain update nightly && rustup default nightly
       - name: Run benchmark
-        run: cd examples/rust && cargo +nightly bench | tee output.txt
+        run: cd examples/benchmarkdotnet && dotnet run --exporters json --filter '*'
 
       - uses: actions/checkout@v4
         with:
@@ -35,30 +37,27 @@ jobs:
       - name: Store benchmark result
         uses: ./
         with:
-          name: Rust Benchmark
-          tool: 'cargo'
-          output-file-path: examples/rust/output.txt
+          name: Benchmark.Net Benchmark
+          tool: 'benchmarkdotnet'
+          output-file-path: examples/benchmarkdotnet/BenchmarkDotNet.Artifacts/results/Sample.Benchmarks-report-full-compressed.json
           fail-on-alert: true
           gh-repository: 'github.com/benchmark-action/github-action-benchmark-results'
           github-token: ${{ secrets.GITHUB_TOKEN }}
-      - run: node ./dist/scripts/ci_validate_modification.js before_data.js 'Rust Benchmark' './benchmark-data-repository'
+      - run: node ./dist/scripts/ci_validate_modification.js before_data.js 'Benchmark.Net Benchmark' './benchmark-data-repository'
 
-  go:
-    name: Run Go benchmark example - github.com/benchmark-action/github-action-benchmark-results
+  benchmarkjs:
+    name: Run JavaScript benchmark example - github.com/benchmark-action/github-action-benchmark-results
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
       - uses: actions/setup-node@v4
         with:
           node-version: 20
           cache: 'npm'
-      - uses: actions/setup-go@v4
-        with:
-          go-version: "stable"
       - run: npm ci
       - run: npm run build
       - name: Run benchmark
-        run: cd examples/go && go test -bench 'BenchmarkFib' | tee output.txt
+        run: cd examples/benchmarkjs && npm install && node bench.js | tee output.txt
 
       - uses: actions/checkout@v4
         with:
@@ -72,17 +71,17 @@ jobs:
       - name: Store benchmark result
         uses: ./
         with:
-          name: Go Benchmark
-          tool: 'go'
-          output-file-path: examples/go/output.txt
+          name: Benchmark.js Benchmark
+          tool: 'benchmarkjs'
+          output-file-path: examples/benchmarkjs/output.txt
           fail-on-alert: true
           gh-repository: 'github.com/benchmark-action/github-action-benchmark-results'
           github-token: ${{ secrets.GITHUB_TOKEN }}
-      - run: node ./dist/scripts/ci_validate_modification.js before_data.js 'Go Benchmark' './benchmark-data-repository'
+      - run: node ./dist/scripts/ci_validate_modification.js before_data.js 'Benchmark.js Benchmark' './benchmark-data-repository'
 
-  benchmarkjs:
-    name: Run JavaScript benchmark example - github.com/benchmark-action/github-action-benchmark-results
-    runs-on: ubuntu-latest
+  catch2-framework:
+    name: Run Catch2 C++ Benchmark Framework example - github.com/benchmark-action/github-action-benchmark-results
+    runs-on: ubuntu-20.04
     steps:
       - uses: actions/checkout@v4
       - uses: actions/setup-node@v4
@@ -92,7 +91,12 @@ jobs:
       - run: npm ci
       - run: npm run build
       - name: Run benchmark
-        run: cd examples/benchmarkjs && npm install && node bench.js | tee output.txt
+        run: |
+          cd examples/catch2
+          mkdir build && cd build
+          cmake -DCMAKE_BUILD_TYPE=Release ..
+          cmake --build . --config Release
+          ./Catch2_bench > ../benchmark_result.txt
 
       - uses: actions/checkout@v4
         with:
@@ -106,33 +110,34 @@ jobs:
       - name: Store benchmark result
         uses: ./
         with:
-          name: Benchmark.js Benchmark
-          tool: 'benchmarkjs'
-          output-file-path: examples/benchmarkjs/output.txt
+          name: Catch2 Benchmark
+          tool: 'catch2'
+          output-file-path: examples/catch2/benchmark_result.txt
           fail-on-alert: true
           gh-repository: 'github.com/benchmark-action/github-action-benchmark-results'
           github-token: ${{ secrets.GITHUB_TOKEN }}
-      - run: node ./dist/scripts/ci_validate_modification.js before_data.js 'Benchmark.js Benchmark' './benchmark-data-repository'
+      - run: node ./dist/scripts/ci_validate_modification.js before_data.js 'Catch2 Benchmark' './benchmark-data-repository'
 
-  pytest-benchmark:
-    name: Run Pytest benchmark example - github.com/benchmark-action/github-action-benchmark-results
-    runs-on: ubuntu-latest
+  cpp-framework:
+    name: Run Google C++ Benchmark Framework example - github.com/benchmark-action/github-action-benchmark-results
+    runs-on: ubuntu-20.04
     steps:
       - uses: actions/checkout@v4
       - uses: actions/setup-node@v4
         with:
           node-version: 20
           cache: 'npm'
-      - uses: actions/setup-python@v5
-        with:
-          python-version: 3.9
       - run: npm ci
       - run: npm run build
+      - name: Cache Benchmark library
+        uses: actions/cache@v4
+        with:
+          path: examples/cpp/benchmark
+          key: ${{ runner.os }}-googlebenchmark-v1.5.0
       - name: Run benchmark
         run: |
-          cd examples/pytest
-          pip install -r requirements.txt
-          pytest bench.py --benchmark-json output.json
+          cd examples/cpp
+          make json
 
       - uses: actions/checkout@v4
         with:
@@ -146,34 +151,30 @@ jobs:
       - name: Store benchmark result
         uses: ./
         with:
-          name: Python Benchmark with pytest-benchmark
-          tool: 'pytest'
-          output-file-path: examples/pytest/output.json
+          name: C++ Benchmark
+          tool: 'googlecpp'
+          output-file-path: examples/cpp/benchmark_result.json
           fail-on-alert: true
           gh-repository: 'github.com/benchmark-action/github-action-benchmark-results'
           github-token: ${{ secrets.GITHUB_TOKEN }}
-      - run: node ./dist/scripts/ci_validate_modification.js before_data.js 'Python Benchmark with pytest-benchmark' './benchmark-data-repository'
+      - run: node ./dist/scripts/ci_validate_modification.js before_data.js 'C++ Benchmark' './benchmark-data-repository'
 
-  google-benchmark-framework:
-    name: Run Google C++ Benchmark Framework example - github.com/benchmark-action/github-action-benchmark-results
-    runs-on: ubuntu-20.04
+  go:
+    name: Run Go benchmark example - github.com/benchmark-action/github-action-benchmark-results
+    runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
       - uses: actions/setup-node@v4
         with:
           node-version: 20
           cache: 'npm'
+      - uses: actions/setup-go@v4
+        with:
+          go-version: "stable"
       - run: npm ci
       - run: npm run build
-      - name: Cache Benchmark library
-        uses: actions/cache@v4
-        with:
-          path: examples/cpp/benchmark
-          key: ${{ runner.os }}-googlebenchmark-v1.5.0
       - name: Run benchmark
-        run: |
-          cd examples/cpp
-          make json
+        run: cd examples/go && go test -bench 'BenchmarkFib' | tee output.txt
 
       - uses: actions/checkout@v4
         with:
@@ -187,33 +188,34 @@ jobs:
       - name: Store benchmark result
         uses: ./
         with:
-          name: C++ Benchmark
-          tool: 'googlecpp'
-          output-file-path: examples/cpp/benchmark_result.json
+          name: Go Benchmark
+          tool: 'go'
+          output-file-path: examples/go/output.txt
           fail-on-alert: true
           gh-repository: 'github.com/benchmark-action/github-action-benchmark-results'
           github-token: ${{ secrets.GITHUB_TOKEN }}
-      - run: node ./dist/scripts/ci_validate_modification.js before_data.js 'C++ Benchmark' './benchmark-data-repository'
+      - run: node ./dist/scripts/ci_validate_modification.js before_data.js 'Go Benchmark' './benchmark-data-repository'
 
-  catch2-framework:
-    name: Run Catch2 C++ Benchmark Framework example - github.com/benchmark-action/github-action-benchmark-results
-    runs-on: ubuntu-20.04
+  java-jmh:
+    name: Run JMH Java Benchmark Framework example
+    runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
       - uses: actions/setup-node@v4
         with:
           node-version: 20
           cache: 'npm'
+      - uses: actions/setup-java@v4
+        with:
+          distribution: 'adopt'
+          java-version: '11'
       - run: npm ci
       - run: npm run build
       - name: Run benchmark
         run: |
-          cd examples/catch2
-          mkdir build && cd build
-          cmake -DCMAKE_BUILD_TYPE=Release ..
-          cmake --build . --config Release
-          ./Catch2_bench > ../benchmark_result.txt
-
+          cd examples/java
+          mvn clean verify
+          java -jar target/benchmarks.jar -wi 1 -i 3 -f 1 -rf json
       - uses: actions/checkout@v4
         with:
           repository: benchmark-action/github-action-benchmark-results
@@ -222,17 +224,16 @@ jobs:
       - name: Save previous data.js
         run: |
           cp ./dist/other-repo/dev/bench/data.js before_data.js
-
       - name: Store benchmark result
         uses: ./
         with:
-          name: Catch2 Benchmark
-          tool: 'catch2'
-          output-file-path: examples/catch2/benchmark_result.txt
+          name: JMH Benchmark
+          tool: "jmh"
+          output-file-path: examples/java/jmh-result.json
           fail-on-alert: true
           gh-repository: 'github.com/benchmark-action/github-action-benchmark-results'
           github-token: ${{ secrets.GITHUB_TOKEN }}
-      - run: node ./dist/scripts/ci_validate_modification.js before_data.js 'Catch2 Benchmark' './benchmark-data-repository'
+      - run: node ./dist/scripts/ci_validate_modification.js before_data.js 'JMH Benchmark' './benchmark-data-repository'
 
   julia-benchmark:
     name: Run Julia benchmark example - github.com/benchmark-action/github-action-benchmark-results
@@ -277,22 +278,60 @@ jobs:
           github-token: ${{ secrets.GITHUB_TOKEN }}
       - run: node ./dist/scripts/ci_validate_modification.js before_data.js 'Julia benchmark' './benchmark-data-repository'
 
-  benchmarkdotnet-framework:
-    name: Run Benchmark.Net .Net Benchmark Framework example - github.com/benchmark-action/github-action-benchmark-results
+  pytest-benchmark:
+    name: Run Pytest benchmark example - github.com/benchmark-action/github-action-benchmark-results
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
-      - uses: actions/setup-dotnet@v3
+      - uses: actions/setup-node@v4
         with:
-          dotnet-version: '6.0.101'  # SDK Version to use. keep in line with examples/benchmarkdotnet/global.json
+          node-version: 20
+          cache: 'npm'
+      - uses: actions/setup-python@v5
+        with:
+          python-version: 3.9
+      - run: npm ci
+      - run: npm run build
+      - name: Run benchmark
+        run: |
+          cd examples/pytest
+          pip install -r requirements.txt
+          pytest bench.py --benchmark-json output.json
+
+      - uses: actions/checkout@v4
+        with:
+          repository: benchmark-action/github-action-benchmark-results
+          ref: 'gh-pages'
+          path: 'dist/other-repo'
+      - name: Save previous data.js
+        run: |
+          cp ./dist/other-repo/dev/bench/data.js before_data.js
+
+      - name: Store benchmark result
+        uses: ./
+        with:
+          name: Python Benchmark with pytest-benchmark
+          tool: 'pytest'
+          output-file-path: examples/pytest/output.json
+          fail-on-alert: true
+          gh-repository: 'github.com/benchmark-action/github-action-benchmark-results'
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+      - run: node ./dist/scripts/ci_validate_modification.js before_data.js 'Python Benchmark with pytest-benchmark' './benchmark-data-repository'
+
+  rust:
+    name: Run Rust benchmark example - github.com/benchmark-action/github-action-benchmark-results
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
       - uses: actions/setup-node@v4
         with:
           node-version: 20
           cache: 'npm'
       - run: npm ci
       - run: npm run build
+      - run: rustup toolchain update nightly && rustup default nightly
       - name: Run benchmark
-        run: cd examples/benchmarkdotnet && dotnet run --exporters json --filter '*'
+        run: cd examples/rust && cargo +nightly bench | tee output.txt
 
       - uses: actions/checkout@v4
         with:
@@ -306,13 +345,48 @@ jobs:
       - name: Store benchmark result
         uses: ./
         with:
-          name: Benchmark.Net Benchmark
-          tool: 'benchmarkdotnet'
-          output-file-path: examples/benchmarkdotnet/BenchmarkDotNet.Artifacts/results/Sample.Benchmarks-report-full-compressed.json
+          name: Rust Benchmark
+          tool: 'cargo'
+          output-file-path: examples/rust/output.txt
           fail-on-alert: true
           gh-repository: 'github.com/benchmark-action/github-action-benchmark-results'
           github-token: ${{ secrets.GITHUB_TOKEN }}
-      - run: node ./dist/scripts/ci_validate_modification.js before_data.js 'Benchmark.Net Benchmark' './benchmark-data-repository'
+      - run: node ./dist/scripts/ci_validate_modification.js before_data.js 'Rust Benchmark' './benchmark-data-repository'
+
+  rust-criterion-rs-framework:
+    name: Run Criterion.rs benchmark example - github.com/benchmark-action/github-action-benchmark-results
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-node@v4
+        with:
+          node-version: 20
+          cache: 'npm'
+      - run: npm ci
+      - run: npm run build
+      - run: rustup toolchain update nightly && rustup default nightly
+      - name: Run benchmark
+        run: cd examples/criterion-rs && cargo +nightly bench -- --output-format bencher | tee output.txt
+
+      - uses: actions/checkout@v4
+        with:
+          repository: benchmark-action/github-action-benchmark-results
+          ref: 'gh-pages'
+          path: 'dist/other-repo'
+      - name: Save previous data.js
+        run: |
+          cp ./dist/other-repo/dev/bench/data.js before_data.js
+
+      - name: Store benchmark result
+        uses: benchmark-action/github-action-benchmark@v1
+        with:
+          name: Criterion.rs Benchmark
+          tool: 'cargo'
+          output-file-path: examples/criterion-rs/output.txt
+          fail-on-alert: true
+          gh-repository: 'github.com/benchmark-action/github-action-benchmark-results'
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+      - run: node ./dist/scripts/ci_validate_modification.js before_data.js 'Criterion.rs Benchmark' './benchmark-data-repository'
 
   only-alert-with-cache:
     name: Run alert check with actions/cache