From 79ceada06b6ef77aac73354278141b7a285adbcc Mon Sep 17 00:00:00 2001 From: Chris Trzesniewski Date: Thu, 25 Jan 2024 16:33:02 +0100 Subject: [PATCH] fix: improve flaky CI runs * move saving of previous benchmarks results further in the workflow files so that there is a smaller window when other action might push new changes --- .github/workflows/ci-results-repo.yml | 143 ++++++++++++++------------ .github/workflows/ci.yml | 74 ++++++------- 2 files changed, 114 insertions(+), 103 deletions(-) diff --git a/.github/workflows/ci-results-repo.yml b/.github/workflows/ci-results-repo.yml index 8479199b2..7358f7d8a 100644 --- a/.github/workflows/ci-results-repo.yml +++ b/.github/workflows/ci-results-repo.yml @@ -13,23 +13,25 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - - uses: actions/checkout@v4 - with: - repository: benchmark-action/github-action-benchmark-results - ref: 'gh-pages' - path: 'dist/other-repo' - uses: actions/setup-node@v4 with: node-version: 20 cache: 'npm' - run: npm ci - run: npm run build - - name: Save previous data.js - run: | - cp ./dist/other-repo/dev/bench/data.js before_data.js - run: rustup toolchain update nightly && rustup default nightly - name: Run benchmark run: cd examples/rust && cargo +nightly bench | tee output.txt + + - uses: actions/checkout@v4 + with: + repository: benchmark-action/github-action-benchmark-results + ref: 'gh-pages' + path: 'dist/other-repo' + - name: Save previous data.js + run: | + cp ./dist/other-repo/dev/bench/data.js before_data.js + - name: Store benchmark result uses: ./ with: @@ -46,11 +48,6 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - - uses: actions/checkout@v4 - with: - repository: benchmark-action/github-action-benchmark-results - ref: 'gh-pages' - path: 'dist/other-repo' - uses: actions/setup-node@v4 with: node-version: 20 @@ -60,11 +57,18 @@ jobs: go-version: "stable" - run: npm ci - run: npm run build + - name: Run benchmark + run: cd examples/go && go test -bench 'BenchmarkFib' | tee output.txt + + - uses: actions/checkout@v4 + with: + repository: benchmark-action/github-action-benchmark-results + ref: 'gh-pages' + path: 'dist/other-repo' - name: Save previous data.js run: | cp ./dist/other-repo/dev/bench/data.js before_data.js - - name: Run benchmark - run: cd examples/go && go test -bench 'BenchmarkFib' | tee output.txt + - name: Store benchmark result uses: ./ with: @@ -81,22 +85,24 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - - uses: actions/checkout@v4 - with: - repository: benchmark-action/github-action-benchmark-results - ref: 'gh-pages' - path: 'dist/other-repo' - uses: actions/setup-node@v4 with: node-version: 20 cache: 'npm' - run: npm ci - run: npm run build + - name: Run benchmark + run: cd examples/benchmarkjs && npm install && node bench.js | tee output.txt + + - uses: actions/checkout@v4 + with: + repository: benchmark-action/github-action-benchmark-results + ref: 'gh-pages' + path: 'dist/other-repo' - name: Save previous data.js run: | cp ./dist/other-repo/dev/bench/data.js before_data.js - - name: Run benchmark - run: cd examples/benchmarkjs && npm install && node bench.js | tee output.txt + - name: Store benchmark result uses: ./ with: @@ -113,11 +119,6 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - - uses: actions/checkout@v4 - with: - repository: benchmark-action/github-action-benchmark-results - ref: 'gh-pages' - path: 'dist/other-repo' - uses: actions/setup-node@v4 with: node-version: 20 @@ -127,14 +128,21 @@ jobs: python-version: 3.9 - run: npm ci - run: npm run build - - name: Save previous data.js - run: | - cp ./dist/other-repo/dev/bench/data.js before_data.js - name: Run benchmark run: | cd examples/pytest pip install -r requirements.txt pytest bench.py --benchmark-json output.json + + - uses: actions/checkout@v4 + with: + repository: benchmark-action/github-action-benchmark-results + ref: 'gh-pages' + path: 'dist/other-repo' + - name: Save previous data.js + run: | + cp ./dist/other-repo/dev/bench/data.js before_data.js + - name: Store benchmark result uses: ./ with: @@ -151,20 +159,12 @@ jobs: runs-on: ubuntu-20.04 steps: - uses: actions/checkout@v4 - - uses: actions/checkout@v4 - with: - repository: benchmark-action/github-action-benchmark-results - ref: 'gh-pages' - path: 'dist/other-repo' - uses: actions/setup-node@v4 with: node-version: 20 cache: 'npm' - run: npm ci - run: npm run build - - name: Save previous data.js - run: | - cp ./dist/other-repo/dev/bench/data.js before_data.js - name: Cache Benchmark library uses: actions/cache@v4 with: @@ -174,6 +174,16 @@ jobs: run: | cd examples/cpp make json + + - uses: actions/checkout@v4 + with: + repository: benchmark-action/github-action-benchmark-results + ref: 'gh-pages' + path: 'dist/other-repo' + - name: Save previous data.js + run: | + cp ./dist/other-repo/dev/bench/data.js before_data.js + - name: Store benchmark result uses: ./ with: @@ -190,20 +200,12 @@ jobs: runs-on: ubuntu-20.04 steps: - uses: actions/checkout@v4 - - uses: actions/checkout@v4 - with: - repository: benchmark-action/github-action-benchmark-results - ref: 'gh-pages' - path: 'dist/other-repo' - uses: actions/setup-node@v4 with: node-version: 20 cache: 'npm' - run: npm ci - run: npm run build - - name: Save previous data.js - run: | - cp ./dist/other-repo/dev/bench/data.js before_data.js - name: Run benchmark run: | cd examples/catch2 @@ -211,6 +213,16 @@ jobs: cmake -DCMAKE_BUILD_TYPE=Release .. cmake --build . --config Release ./Catch2_bench > ../benchmark_result.txt + + - uses: actions/checkout@v4 + with: + repository: benchmark-action/github-action-benchmark-results + ref: 'gh-pages' + path: 'dist/other-repo' + - name: Save previous data.js + run: | + cp ./dist/other-repo/dev/bench/data.js before_data.js + - name: Store benchmark result uses: ./ with: @@ -227,20 +239,12 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - - uses: actions/checkout@v4 - with: - repository: benchmark-action/github-action-benchmark-results - ref: 'gh-pages' - path: 'dist/other-repo' - uses: actions/setup-node@v4 with: node-version: 20 cache: 'npm' - run: npm ci - run: npm run build - - name: Save previous data.js - run: | - cp ./dist/other-repo/dev/bench/data.js before_data.js - uses: julia-actions/setup-julia@v1 with: version: '1' @@ -252,6 +256,16 @@ jobs: using Pkg; Pkg.instantiate(); include("fib.jl")' + + - uses: actions/checkout@v4 + with: + repository: benchmark-action/github-action-benchmark-results + ref: 'gh-pages' + path: 'dist/other-repo' + - name: Save previous data.js + run: | + cp ./dist/other-repo/dev/bench/data.js before_data.js + - name: Store benchmark result uses: ./ with: @@ -268,11 +282,6 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - - uses: actions/checkout@v4 - with: - repository: benchmark-action/github-action-benchmark-results - ref: 'gh-pages' - path: 'dist/other-repo' - uses: actions/setup-dotnet@v3 with: dotnet-version: '6.0.101' # SDK Version to use. keep in line with examples/benchmarkdotnet/global.json @@ -282,11 +291,18 @@ jobs: cache: 'npm' - run: npm ci - run: npm run build + - name: Run benchmark + run: cd examples/benchmarkdotnet && dotnet run --exporters json --filter '*' + + - uses: actions/checkout@v4 + with: + repository: benchmark-action/github-action-benchmark-results + ref: 'gh-pages' + path: 'dist/other-repo' - name: Save previous data.js run: | cp ./dist/other-repo/dev/bench/data.js before_data.js - - name: Run benchmark - run: cd examples/benchmarkdotnet && dotnet run --exporters json --filter '*' + - name: Store benchmark result uses: ./ with: @@ -303,11 +319,6 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - - uses: actions/checkout@v4 - with: - repository: benchmark-action/github-action-benchmark-results - ref: 'gh-pages' - path: 'dist/other-repo' - uses: actions/setup-node@v4 with: node-version: 20 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 597cc8e24..006ed60d0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -19,15 +19,15 @@ jobs: cache: 'npm' - run: npm ci - run: npm run build + - run: rustup toolchain update nightly && rustup default nightly + - name: Run benchmark + run: cd examples/rust && cargo +nightly bench | tee output.txt - name: Save previous data.js run: | git fetch origin gh-pages git checkout gh-pages cp ./dev/bench/data.js before_data.js git checkout - - - run: rustup toolchain update nightly && rustup default nightly - - name: Run benchmark - run: cd examples/rust && cargo +nightly bench | tee output.txt - name: Store benchmark result uses: ./ with: @@ -52,14 +52,14 @@ jobs: go-version: "stable" - run: npm ci - run: npm run build + - name: Run benchmark + run: cd examples/go && go test -bench 'BenchmarkFib' | tee output.txt - name: Save previous data.js run: | git fetch origin gh-pages git checkout gh-pages cp ./dev/bench/data.js before_data.js git checkout - - - name: Run benchmark - run: cd examples/go && go test -bench 'BenchmarkFib' | tee output.txt - name: Store benchmark result uses: ./ with: @@ -81,14 +81,14 @@ jobs: cache: 'npm' - run: npm ci - run: npm run build + - name: Run benchmark + run: cd examples/benchmarkjs && npm install && node bench.js | tee output.txt - name: Save previous data.js run: | git fetch origin gh-pages git checkout gh-pages cp ./dev/bench/data.js before_data.js git checkout - - - name: Run benchmark - run: cd examples/benchmarkjs && npm install && node bench.js | tee output.txt - name: Store benchmark result uses: ./ with: @@ -113,17 +113,17 @@ jobs: python-version: 3.9 - run: npm ci - run: npm run build + - name: Run benchmark + run: | + cd examples/pytest + pip install -r requirements.txt + pytest bench.py --benchmark-json output.json - name: Save previous data.js run: | git fetch origin gh-pages git checkout gh-pages cp ./dev/bench/data.js before_data.js git checkout - - - name: Run benchmark - run: | - cd examples/pytest - pip install -r requirements.txt - pytest bench.py --benchmark-json output.json - name: Store benchmark result uses: ./ with: @@ -145,12 +145,6 @@ jobs: cache: 'npm' - run: npm ci - run: npm run build - - name: Save previous data.js - run: | - git fetch origin gh-pages - git checkout gh-pages - cp ./dev/bench/data.js before_data.js - git checkout - - name: Cache Benchmark library uses: actions/cache@v4 with: @@ -160,6 +154,12 @@ jobs: run: | cd examples/cpp make json + - name: Save previous data.js + run: | + git fetch origin gh-pages + git checkout gh-pages + cp ./dev/bench/data.js before_data.js + git checkout - - name: Store benchmark result uses: ./ with: @@ -181,12 +181,6 @@ jobs: cache: 'npm' - run: npm ci - run: npm run build - - name: Save previous data.js - run: | - git fetch origin gh-pages - git checkout gh-pages - cp ./dev/bench/data.js before_data.js - git checkout - - name: Run benchmark run: | cd examples/catch2 @@ -194,6 +188,12 @@ jobs: cmake -DCMAKE_BUILD_TYPE=Release .. cmake --build . --config Release ./Catch2_bench > ../benchmark_result.txt + - name: Save previous data.js + run: | + git fetch origin gh-pages + git checkout gh-pages + cp ./dev/bench/data.js before_data.js + git checkout - - name: Store benchmark result uses: ./ with: @@ -215,12 +215,6 @@ jobs: cache: 'npm' - run: npm ci - run: npm run build - - name: Save previous data.js - run: | - git fetch origin gh-pages - git checkout gh-pages - cp ./dev/bench/data.js before_data.js - git checkout - - uses: julia-actions/setup-julia@v1 with: version: '1' @@ -232,6 +226,12 @@ jobs: using Pkg; Pkg.instantiate(); include("fib.jl")' + - name: Save previous data.js + run: | + git fetch origin gh-pages + git checkout gh-pages + cp ./dev/bench/data.js before_data.js + git checkout - - name: Store benchmark result uses: ./ with: @@ -256,14 +256,14 @@ jobs: dotnet-version: '6.0.101' # SDK Version to use. keep in line with examples/benchmarkdotnet/global.json - run: npm ci - run: npm run build + - name: Run benchmark + run: cd examples/benchmarkdotnet && dotnet run --exporters json --filter '*' - name: Save previous data.js run: | git fetch origin gh-pages git checkout gh-pages cp ./dev/bench/data.js before_data.js git checkout - - - name: Run benchmark - run: cd examples/benchmarkdotnet && dotnet run --exporters json --filter '*' - name: Store benchmark result uses: ./ with: @@ -289,17 +289,17 @@ jobs: java-version: '11' - run: npm ci - run: npm run build + - name: Run benchmark + run: | + cd examples/java + mvn clean verify + java -jar target/benchmarks.jar -wi 1 -i 3 -f 1 -rf json - name: Save previous data.js run: | git fetch origin gh-pages git checkout gh-pages cp ./dev/bench/data.js before_data.js git checkout - - - name: Run benchmark - run: | - cd examples/java - mvn clean verify - java -jar target/benchmarks.jar -wi 1 -i 3 -f 1 -rf json - name: Store benchmark result uses: ./ with: