From 8335d428dae7d9fa40d49a98693fd6449824cd0e Mon Sep 17 00:00:00 2001 From: Chris Trzesniewski Date: Wed, 31 Jan 2024 15:32:15 +0100 Subject: [PATCH 01/34] feat: auto update PR comment * create PR comment when PR is available --- .github/workflows/ci.yml | 1 + src/write.ts | 23 ++++++++++++++++------- 2 files changed, 17 insertions(+), 7 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index eed878cb8..dc096715a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -36,6 +36,7 @@ jobs: output-file-path: examples/rust/output.txt fail-on-alert: true summary-always: true + comment-always: true - run: node ./dist/scripts/ci_validate_modification.js before_data.js 'Rust Benchmark' go: diff --git a/src/write.ts b/src/write.ts index 3c71823a7..08ce31b94 100644 --- a/src/write.ts +++ b/src/write.ts @@ -241,14 +241,23 @@ async function leaveComment(commitId: string, body: string, token: string) { const repoMetadata = getCurrentRepoMetadata(); const repoUrl = repoMetadata.html_url ?? ''; + const pr = github.context.payload.pull_request; const client = github.getOctokit(token); - const res = await client.rest.repos.createCommitComment({ - owner: repoMetadata.owner.login, - repo: repoMetadata.name, - // eslint-disable-next-line @typescript-eslint/naming-convention - commit_sha: commitId, - body, - }); + const res = await (pr?.number + ? client.rest.pulls.createReviewComment({ + owner: repoMetadata.owner.login, + repo: repoMetadata.name, + // eslint-disable-next-line @typescript-eslint/naming-convention + pull_number: pr.number, + body, + }) + : client.rest.repos.createCommitComment({ + owner: repoMetadata.owner.login, + repo: repoMetadata.name, + // eslint-disable-next-line @typescript-eslint/naming-convention + commit_sha: commitId, + body, + })); const commitUrl = `${repoUrl}/commit/${commitId}`; console.log(`Comment was sent to ${commitUrl}. Response:`, res.status, res.data); From 4b06822719765e70727276fb54637a6939036771 Mon Sep 17 00:00:00 2001 From: Chris Trzesniewski Date: Wed, 31 Jan 2024 15:34:23 +0100 Subject: [PATCH 02/34] feat: auto update PR comment * add gh token to workflow --- .github/workflows/ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index dc096715a..bfca0df03 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -34,6 +34,7 @@ jobs: name: Rust Benchmark tool: 'cargo' output-file-path: examples/rust/output.txt + github-token: ${{ secrets.GITHUB_TOKEN }} fail-on-alert: true summary-always: true comment-always: true From 91dda8a29b06b6aeff5519af09a61ff6e38f2c6d Mon Sep 17 00:00:00 2001 From: Chris Trzesniewski Date: Wed, 31 Jan 2024 15:39:25 +0100 Subject: [PATCH 03/34] feat: auto update PR comment * create review instead of review comment --- src/write.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/write.ts b/src/write.ts index 08ce31b94..a12810811 100644 --- a/src/write.ts +++ b/src/write.ts @@ -244,7 +244,7 @@ async function leaveComment(commitId: string, body: string, token: string) { const pr = github.context.payload.pull_request; const client = github.getOctokit(token); const res = await (pr?.number - ? client.rest.pulls.createReviewComment({ + ? client.rest.pulls.createReview({ owner: repoMetadata.owner.login, repo: repoMetadata.name, // eslint-disable-next-line @typescript-eslint/naming-convention From d01fbb8bc21f5e9fad2ae1ac86482999a81a06e2 Mon Sep 17 00:00:00 2001 From: Chris Trzesniewski Date: Wed, 31 Jan 2024 15:39:59 +0100 Subject: [PATCH 04/34] feat: auto update PR comment * disable unnecessary ci runs --- .github/workflows/ci-results-repo.yml | 6 +- .github/workflows/ci.yml | 684 +++++++++++++------------- 2 files changed, 345 insertions(+), 345 deletions(-) diff --git a/.github/workflows/ci-results-repo.yml b/.github/workflows/ci-results-repo.yml index 7358f7d8a..8f72ee1e4 100644 --- a/.github/workflows/ci-results-repo.yml +++ b/.github/workflows/ci-results-repo.yml @@ -3,9 +3,9 @@ on: push: branches: - master - pull_request: - branches: - - master +# pull_request: +# branches: +# - master jobs: rust: diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index bfca0df03..c46f8037c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -40,345 +40,345 @@ jobs: comment-always: true - run: node ./dist/scripts/ci_validate_modification.js before_data.js 'Rust Benchmark' - go: - name: Run Go benchmark example - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-node@v4 - with: - node-version: 20 - cache: 'npm' - - uses: actions/setup-go@v4 - with: - go-version: "stable" - - run: npm ci - - run: npm run build - - name: Run benchmark - run: cd examples/go && go test -bench 'BenchmarkFib' | tee output.txt - - name: Save previous data.js - run: | - git fetch origin gh-pages - git checkout gh-pages - cp ./dev/bench/data.js before_data.js - git checkout - - - name: Store benchmark result - uses: ./ - with: - name: Go Benchmark - tool: 'go' - output-file-path: examples/go/output.txt - skip-fetch-gh-pages: true - fail-on-alert: true - summary-always: true - - run: node ./dist/scripts/ci_validate_modification.js before_data.js 'Go Benchmark' - - benchmarkjs: - name: Run JavaScript benchmark example - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-node@v4 - with: - node-version: 20 - cache: 'npm' - - run: npm ci - - run: npm run build - - name: Run benchmark - run: cd examples/benchmarkjs && npm install && node bench.js | tee output.txt - - name: Save previous data.js - run: | - git fetch origin gh-pages - git checkout gh-pages - cp ./dev/bench/data.js before_data.js - git checkout - - - name: Store benchmark result - uses: ./ - with: - name: Benchmark.js Benchmark - tool: 'benchmarkjs' - output-file-path: examples/benchmarkjs/output.txt - skip-fetch-gh-pages: true - fail-on-alert: true - summary-always: true - - run: node ./dist/scripts/ci_validate_modification.js before_data.js 'Benchmark.js Benchmark' - - pytest-benchmark: - name: Run Pytest benchmark example - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-node@v4 - with: - node-version: 20 - cache: 'npm' - - uses: actions/setup-python@v5 - with: - python-version: 3.9 - - run: npm ci - - run: npm run build - - name: Run benchmark - run: | - cd examples/pytest - pip install -r requirements.txt - pytest bench.py --benchmark-json output.json - - name: Save previous data.js - run: | - git fetch origin gh-pages - git checkout gh-pages - cp ./dev/bench/data.js before_data.js - git checkout - - - name: Store benchmark result - uses: ./ - with: - name: Python Benchmark with pytest-benchmark - tool: 'pytest' - output-file-path: examples/pytest/output.json - skip-fetch-gh-pages: true - fail-on-alert: true - summary-always: true - - run: node ./dist/scripts/ci_validate_modification.js before_data.js 'Python Benchmark with pytest-benchmark' - - google-benchmark-framework: - name: Run Google C++ Benchmark Framework example - runs-on: ubuntu-20.04 - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-node@v4 - with: - node-version: 20 - cache: 'npm' - - run: npm ci - - run: npm run build - - name: Cache Benchmark library - uses: actions/cache@v4 - with: - path: examples/cpp/benchmark - key: ${{ runner.os }}-googlebenchmark-v1.5.0 - - name: Run benchmark - run: | - cd examples/cpp - make json - - name: Save previous data.js - run: | - git fetch origin gh-pages - git checkout gh-pages - cp ./dev/bench/data.js before_data.js - git checkout - - - name: Store benchmark result - uses: ./ - with: - name: C++ Benchmark - tool: 'googlecpp' - output-file-path: examples/cpp/benchmark_result.json - skip-fetch-gh-pages: true - fail-on-alert: true - summary-always: true - - run: node ./dist/scripts/ci_validate_modification.js before_data.js 'C++ Benchmark' - - catch2-framework: - name: Run Catch2 C++ Benchmark Framework example - runs-on: ubuntu-20.04 - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-node@v4 - with: - node-version: 20 - cache: 'npm' - - run: npm ci - - run: npm run build - - name: Run benchmark - run: | - cd examples/catch2 - mkdir build && cd build - cmake -DCMAKE_BUILD_TYPE=Release .. - cmake --build . --config Release - ./Catch2_bench > ../benchmark_result.txt - - name: Save previous data.js - run: | - git fetch origin gh-pages - git checkout gh-pages - cp ./dev/bench/data.js before_data.js - git checkout - - - name: Store benchmark result - uses: ./ - with: - name: Catch2 Benchmark - tool: 'catch2' - output-file-path: examples/catch2/benchmark_result.txt - skip-fetch-gh-pages: true - fail-on-alert: true - summary-always: true - - run: node ./dist/scripts/ci_validate_modification.js before_data.js 'Catch2 Benchmark' - - julia-benchmark: - name: Run Julia benchmark example - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-node@v4 - with: - node-version: 20 - cache: 'npm' - - run: npm ci - - run: npm run build - - uses: julia-actions/setup-julia@v1 - with: - version: '1' - arch: x64 - - name: Run benchmark - run: | - cd examples/julia - julia --project --color=yes -e ' - using Pkg; - Pkg.instantiate(); - include("fib.jl")' - - name: Save previous data.js - run: | - git fetch origin gh-pages - git checkout gh-pages - cp ./dev/bench/data.js before_data.js - git checkout - - - name: Store benchmark result - uses: ./ - with: - name: Julia benchmark - tool: 'julia' - output-file-path: examples/julia/output.json - skip-fetch-gh-pages: true - fail-on-alert: true - summary-always: true - - run: node ./dist/scripts/ci_validate_modification.js before_data.js 'Julia benchmark' - - benchmarkdotnet-framework: - name: Run Benchmark.Net .Net Benchmark Framework example - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-node@v4 - with: - node-version: 20 - cache: 'npm' - - uses: actions/setup-dotnet@v3 - with: - dotnet-version: '6.0.101' # SDK Version to use. keep in line with examples/benchmarkdotnet/global.json - - run: npm ci - - run: npm run build - - name: Run benchmark - run: cd examples/benchmarkdotnet && dotnet run --exporters json --filter '*' - - name: Save previous data.js - run: | - git fetch origin gh-pages - git checkout gh-pages - cp ./dev/bench/data.js before_data.js - git checkout - - - name: Store benchmark result - uses: ./ - with: - name: Benchmark.Net Benchmark - tool: 'benchmarkdotnet' - output-file-path: examples/benchmarkdotnet/BenchmarkDotNet.Artifacts/results/Sample.Benchmarks-report-full-compressed.json - skip-fetch-gh-pages: true - fail-on-alert: true - summary-always: true - - run: node ./dist/scripts/ci_validate_modification.js before_data.js 'Benchmark.Net Benchmark' - - jmh: - name: Run JMH Java Benchmark Framework example - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-node@v4 - with: - node-version: 20 - cache: 'npm' - - uses: actions/setup-java@v4 - with: - distribution: 'adopt' - java-version: '11' - - run: npm ci - - run: npm run build - - name: Run benchmark - run: | - cd examples/java - mvn clean verify - java -jar target/benchmarks.jar -wi 1 -i 3 -f 1 -rf json - - name: Save previous data.js - run: | - git fetch origin gh-pages - git checkout gh-pages - cp ./dev/bench/data.js before_data.js - git checkout - - - name: Store benchmark result - uses: ./ - with: - name: JMH Benchmark - tool: "jmh" - output-file-path: examples/java/jmh-result.json - skip-fetch-gh-pages: true - fail-on-alert: true - summary-always: true - - run: node ./dist/scripts/ci_validate_modification.js before_data.js 'JMH Benchmark' - - only-alert-with-cache: - name: Run alert check with actions/cache - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-node@v4 - with: - node-version: 20 - cache: 'npm' - - run: npm ci - - run: npm run build - - uses: actions/setup-go@v4 - with: - go-version: "stable" - - name: Run benchmark - run: cd examples/go && go test -bench 'BenchmarkFib' | tee output.txt - - name: Download previous benchmark data - uses: actions/cache@v4 - with: - path: ./cache - key: ${{ runner.os }}-ci-cache-test - - name: Store benchmark result - uses: ./ - with: - name: Alert setup example with cache - tool: 'go' - output-file-path: examples/go/output.txt - external-data-json-path: ./cache/benchmark-data.json - github-token: ${{ secrets.GITHUB_TOKEN }} - alert-threshold: '200%' - comment-on-alert: true - fail-on-alert: true - alert-comment-cc-users: '@ktrz' - unit-tests: - name: Run unit tests - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-node@v4 - with: - node-version: 20 - cache: 'npm' - - run: npm ci - - run: npm run coverage - - name: Apply yamllint - run: | - sudo pip install yamllint - yamllint --strict .github/workflows - - uses: codecov/codecov-action@v2 - - lint: - name: Run linting and formatting check - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-node@v4 - with: - node-version: 20 - cache: 'npm' - - run: npm ci - - run: npm run lint - - run: npm run format:check +# go: +# name: Run Go benchmark example +# runs-on: ubuntu-latest +# steps: +# - uses: actions/checkout@v4 +# - uses: actions/setup-node@v4 +# with: +# node-version: 20 +# cache: 'npm' +# - uses: actions/setup-go@v4 +# with: +# go-version: "stable" +# - run: npm ci +# - run: npm run build +# - name: Run benchmark +# run: cd examples/go && go test -bench 'BenchmarkFib' | tee output.txt +# - name: Save previous data.js +# run: | +# git fetch origin gh-pages +# git checkout gh-pages +# cp ./dev/bench/data.js before_data.js +# git checkout - +# - name: Store benchmark result +# uses: ./ +# with: +# name: Go Benchmark +# tool: 'go' +# output-file-path: examples/go/output.txt +# skip-fetch-gh-pages: true +# fail-on-alert: true +# summary-always: true +# - run: node ./dist/scripts/ci_validate_modification.js before_data.js 'Go Benchmark' +# +# benchmarkjs: +# name: Run JavaScript benchmark example +# runs-on: ubuntu-latest +# steps: +# - uses: actions/checkout@v4 +# - uses: actions/setup-node@v4 +# with: +# node-version: 20 +# cache: 'npm' +# - run: npm ci +# - run: npm run build +# - name: Run benchmark +# run: cd examples/benchmarkjs && npm install && node bench.js | tee output.txt +# - name: Save previous data.js +# run: | +# git fetch origin gh-pages +# git checkout gh-pages +# cp ./dev/bench/data.js before_data.js +# git checkout - +# - name: Store benchmark result +# uses: ./ +# with: +# name: Benchmark.js Benchmark +# tool: 'benchmarkjs' +# output-file-path: examples/benchmarkjs/output.txt +# skip-fetch-gh-pages: true +# fail-on-alert: true +# summary-always: true +# - run: node ./dist/scripts/ci_validate_modification.js before_data.js 'Benchmark.js Benchmark' +# +# pytest-benchmark: +# name: Run Pytest benchmark example +# runs-on: ubuntu-latest +# steps: +# - uses: actions/checkout@v4 +# - uses: actions/setup-node@v4 +# with: +# node-version: 20 +# cache: 'npm' +# - uses: actions/setup-python@v5 +# with: +# python-version: 3.9 +# - run: npm ci +# - run: npm run build +# - name: Run benchmark +# run: | +# cd examples/pytest +# pip install -r requirements.txt +# pytest bench.py --benchmark-json output.json +# - name: Save previous data.js +# run: | +# git fetch origin gh-pages +# git checkout gh-pages +# cp ./dev/bench/data.js before_data.js +# git checkout - +# - name: Store benchmark result +# uses: ./ +# with: +# name: Python Benchmark with pytest-benchmark +# tool: 'pytest' +# output-file-path: examples/pytest/output.json +# skip-fetch-gh-pages: true +# fail-on-alert: true +# summary-always: true +# - run: node ./dist/scripts/ci_validate_modification.js before_data.js 'Python Benchmark with pytest-benchmark' +# +# google-benchmark-framework: +# name: Run Google C++ Benchmark Framework example +# runs-on: ubuntu-20.04 +# steps: +# - uses: actions/checkout@v4 +# - uses: actions/setup-node@v4 +# with: +# node-version: 20 +# cache: 'npm' +# - run: npm ci +# - run: npm run build +# - name: Cache Benchmark library +# uses: actions/cache@v4 +# with: +# path: examples/cpp/benchmark +# key: ${{ runner.os }}-googlebenchmark-v1.5.0 +# - name: Run benchmark +# run: | +# cd examples/cpp +# make json +# - name: Save previous data.js +# run: | +# git fetch origin gh-pages +# git checkout gh-pages +# cp ./dev/bench/data.js before_data.js +# git checkout - +# - name: Store benchmark result +# uses: ./ +# with: +# name: C++ Benchmark +# tool: 'googlecpp' +# output-file-path: examples/cpp/benchmark_result.json +# skip-fetch-gh-pages: true +# fail-on-alert: true +# summary-always: true +# - run: node ./dist/scripts/ci_validate_modification.js before_data.js 'C++ Benchmark' +# +# catch2-framework: +# name: Run Catch2 C++ Benchmark Framework example +# runs-on: ubuntu-20.04 +# steps: +# - uses: actions/checkout@v4 +# - uses: actions/setup-node@v4 +# with: +# node-version: 20 +# cache: 'npm' +# - run: npm ci +# - run: npm run build +# - name: Run benchmark +# run: | +# cd examples/catch2 +# mkdir build && cd build +# cmake -DCMAKE_BUILD_TYPE=Release .. +# cmake --build . --config Release +# ./Catch2_bench > ../benchmark_result.txt +# - name: Save previous data.js +# run: | +# git fetch origin gh-pages +# git checkout gh-pages +# cp ./dev/bench/data.js before_data.js +# git checkout - +# - name: Store benchmark result +# uses: ./ +# with: +# name: Catch2 Benchmark +# tool: 'catch2' +# output-file-path: examples/catch2/benchmark_result.txt +# skip-fetch-gh-pages: true +# fail-on-alert: true +# summary-always: true +# - run: node ./dist/scripts/ci_validate_modification.js before_data.js 'Catch2 Benchmark' +# +# julia-benchmark: +# name: Run Julia benchmark example +# runs-on: ubuntu-latest +# steps: +# - uses: actions/checkout@v4 +# - uses: actions/setup-node@v4 +# with: +# node-version: 20 +# cache: 'npm' +# - run: npm ci +# - run: npm run build +# - uses: julia-actions/setup-julia@v1 +# with: +# version: '1' +# arch: x64 +# - name: Run benchmark +# run: | +# cd examples/julia +# julia --project --color=yes -e ' +# using Pkg; +# Pkg.instantiate(); +# include("fib.jl")' +# - name: Save previous data.js +# run: | +# git fetch origin gh-pages +# git checkout gh-pages +# cp ./dev/bench/data.js before_data.js +# git checkout - +# - name: Store benchmark result +# uses: ./ +# with: +# name: Julia benchmark +# tool: 'julia' +# output-file-path: examples/julia/output.json +# skip-fetch-gh-pages: true +# fail-on-alert: true +# summary-always: true +# - run: node ./dist/scripts/ci_validate_modification.js before_data.js 'Julia benchmark' +# +# benchmarkdotnet-framework: +# name: Run Benchmark.Net .Net Benchmark Framework example +# runs-on: ubuntu-latest +# steps: +# - uses: actions/checkout@v4 +# - uses: actions/setup-node@v4 +# with: +# node-version: 20 +# cache: 'npm' +# - uses: actions/setup-dotnet@v3 +# with: +# dotnet-version: '6.0.101' # SDK Version to use. keep in line with examples/benchmarkdotnet/global.json +# - run: npm ci +# - run: npm run build +# - name: Run benchmark +# run: cd examples/benchmarkdotnet && dotnet run --exporters json --filter '*' +# - name: Save previous data.js +# run: | +# git fetch origin gh-pages +# git checkout gh-pages +# cp ./dev/bench/data.js before_data.js +# git checkout - +# - name: Store benchmark result +# uses: ./ +# with: +# name: Benchmark.Net Benchmark +# tool: 'benchmarkdotnet' +# output-file-path: examples/benchmarkdotnet/BenchmarkDotNet.Artifacts/results/Sample.Benchmarks-report-full-compressed.json +# skip-fetch-gh-pages: true +# fail-on-alert: true +# summary-always: true +# - run: node ./dist/scripts/ci_validate_modification.js before_data.js 'Benchmark.Net Benchmark' +# +# jmh: +# name: Run JMH Java Benchmark Framework example +# runs-on: ubuntu-latest +# steps: +# - uses: actions/checkout@v4 +# - uses: actions/setup-node@v4 +# with: +# node-version: 20 +# cache: 'npm' +# - uses: actions/setup-java@v4 +# with: +# distribution: 'adopt' +# java-version: '11' +# - run: npm ci +# - run: npm run build +# - name: Run benchmark +# run: | +# cd examples/java +# mvn clean verify +# java -jar target/benchmarks.jar -wi 1 -i 3 -f 1 -rf json +# - name: Save previous data.js +# run: | +# git fetch origin gh-pages +# git checkout gh-pages +# cp ./dev/bench/data.js before_data.js +# git checkout - +# - name: Store benchmark result +# uses: ./ +# with: +# name: JMH Benchmark +# tool: "jmh" +# output-file-path: examples/java/jmh-result.json +# skip-fetch-gh-pages: true +# fail-on-alert: true +# summary-always: true +# - run: node ./dist/scripts/ci_validate_modification.js before_data.js 'JMH Benchmark' +# +# only-alert-with-cache: +# name: Run alert check with actions/cache +# runs-on: ubuntu-latest +# steps: +# - uses: actions/checkout@v4 +# - uses: actions/setup-node@v4 +# with: +# node-version: 20 +# cache: 'npm' +# - run: npm ci +# - run: npm run build +# - uses: actions/setup-go@v4 +# with: +# go-version: "stable" +# - name: Run benchmark +# run: cd examples/go && go test -bench 'BenchmarkFib' | tee output.txt +# - name: Download previous benchmark data +# uses: actions/cache@v4 +# with: +# path: ./cache +# key: ${{ runner.os }}-ci-cache-test +# - name: Store benchmark result +# uses: ./ +# with: +# name: Alert setup example with cache +# tool: 'go' +# output-file-path: examples/go/output.txt +# external-data-json-path: ./cache/benchmark-data.json +# github-token: ${{ secrets.GITHUB_TOKEN }} +# alert-threshold: '200%' +# comment-on-alert: true +# fail-on-alert: true +# alert-comment-cc-users: '@ktrz' +# unit-tests: +# name: Run unit tests +# runs-on: ubuntu-latest +# steps: +# - uses: actions/checkout@v4 +# - uses: actions/setup-node@v4 +# with: +# node-version: 20 +# cache: 'npm' +# - run: npm ci +# - run: npm run coverage +# - name: Apply yamllint +# run: | +# sudo pip install yamllint +# yamllint --strict .github/workflows +# - uses: codecov/codecov-action@v2 +# +# lint: +# name: Run linting and formatting check +# runs-on: ubuntu-latest +# steps: +# - uses: actions/checkout@v4 +# - uses: actions/setup-node@v4 +# with: +# node-version: 20 +# cache: 'npm' +# - run: npm ci +# - run: npm run lint +# - run: npm run format:check From d647ff1b41fa6d28e7aa62a6e32e9d43932faaf9 Mon Sep 17 00:00:00 2001 From: Chris Trzesniewski Date: Wed, 31 Jan 2024 15:46:42 +0100 Subject: [PATCH 05/34] feat: auto update PR comment * add pr write permissions --- .github/workflows/ci.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c46f8037c..44ebeada5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -7,6 +7,9 @@ on: branches: - master +permissions: + pull-requests: write + jobs: rust: name: Run Rust benchmark example From 0ee988e03109cfa450aecbf0c3d584e80db53372 Mon Sep 17 00:00:00 2001 From: Chris Trzesniewski Date: Wed, 31 Jan 2024 15:58:00 +0100 Subject: [PATCH 06/34] feat: auto update PR comment * remove pending review --- src/write.ts | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/src/write.ts b/src/write.ts index a12810811..1c520027e 100644 --- a/src/write.ts +++ b/src/write.ts @@ -243,12 +243,35 @@ async function leaveComment(commitId: string, body: string, token: string) { const repoUrl = repoMetadata.html_url ?? ''; const pr = github.context.payload.pull_request; const client = github.getOctokit(token); + const reviews = await client.rest.pulls.listReviews({ + owner: repoMetadata.owner.login, + repo: repoMetadata.name, + // eslint-disable-next-line @typescript-eslint/naming-convention + pull_number: pr?.number ?? 0, + }); + + await Promise.all( + reviews.data + .filter((r) => r.state === 'PENDING') + .map((r) => + client.rest.pulls.deletePendingReview({ + owner: repoMetadata.owner.login, + repo: repoMetadata.name, + // eslint-disable-next-line @typescript-eslint/naming-convention + pull_number: pr?.number ?? 0, + // eslint-disable-next-line @typescript-eslint/naming-convention + review_id: r.id, + }), + ), + ); + const res = await (pr?.number ? client.rest.pulls.createReview({ owner: repoMetadata.owner.login, repo: repoMetadata.name, // eslint-disable-next-line @typescript-eslint/naming-convention pull_number: pr.number, + event: 'COMMENT', body, }) : client.rest.repos.createCommitComment({ From d830ad7a739df2e7f58dd17e02e2345c0b1685a6 Mon Sep 17 00:00:00 2001 From: Chris Trzesniewski Date: Fri, 2 Feb 2024 13:41:48 +0100 Subject: [PATCH 07/34] feat: auto update PR comment * revert removing pending review --- src/write.ts | 21 --------------------- 1 file changed, 21 deletions(-) diff --git a/src/write.ts b/src/write.ts index 1c520027e..9cd04189c 100644 --- a/src/write.ts +++ b/src/write.ts @@ -243,27 +243,6 @@ async function leaveComment(commitId: string, body: string, token: string) { const repoUrl = repoMetadata.html_url ?? ''; const pr = github.context.payload.pull_request; const client = github.getOctokit(token); - const reviews = await client.rest.pulls.listReviews({ - owner: repoMetadata.owner.login, - repo: repoMetadata.name, - // eslint-disable-next-line @typescript-eslint/naming-convention - pull_number: pr?.number ?? 0, - }); - - await Promise.all( - reviews.data - .filter((r) => r.state === 'PENDING') - .map((r) => - client.rest.pulls.deletePendingReview({ - owner: repoMetadata.owner.login, - repo: repoMetadata.name, - // eslint-disable-next-line @typescript-eslint/naming-convention - pull_number: pr?.number ?? 0, - // eslint-disable-next-line @typescript-eslint/naming-convention - review_id: r.id, - }), - ), - ); const res = await (pr?.number ? client.rest.pulls.createReview({ From b92acc365a98cf53d1a1e75e3eb69b387109f1ec Mon Sep 17 00:00:00 2001 From: Chris Trzesniewski Date: Fri, 2 Feb 2024 14:20:13 +0100 Subject: [PATCH 08/34] feat: auto update PR comment * add logic for updating comment --- src/comment/benchmarkCommentTags.ts | 13 +++++++ src/comment/findExistingPRReviewId.ts | 25 ++++++++++++ src/comment/leaveCommitComment.ts | 22 +++++++++++ src/comment/leavePRComment.ts | 56 +++++++++++++++++++++++++++ src/write.ts | 36 +++++------------ 5 files changed, 125 insertions(+), 27 deletions(-) create mode 100644 src/comment/benchmarkCommentTags.ts create mode 100644 src/comment/findExistingPRReviewId.ts create mode 100644 src/comment/leaveCommitComment.ts create mode 100644 src/comment/leavePRComment.ts diff --git a/src/comment/benchmarkCommentTags.ts b/src/comment/benchmarkCommentTags.ts new file mode 100644 index 000000000..6e88bffb2 --- /dev/null +++ b/src/comment/benchmarkCommentTags.ts @@ -0,0 +1,13 @@ +const BENCHMARK_COMMENT_TAG = 'github-benchmark-action-comment'; + +export function benchmarkStartTag(commentId: string) { + return ``; +} + +export function benchmarkEndTag(commentId: string) { + return ``; +} + +export function wrapBodyWithBenchmarkTags(commentId: string, body: string) { + return `${benchmarkStartTag(commentId)}\n${body}\n${benchmarkEndTag(commentId)}`; +} diff --git a/src/comment/findExistingPRReviewId.ts b/src/comment/findExistingPRReviewId.ts new file mode 100644 index 000000000..1b015e64a --- /dev/null +++ b/src/comment/findExistingPRReviewId.ts @@ -0,0 +1,25 @@ +import * as github from '@actions/github'; +import { benchmarkStartTag } from './benchmarkCommentTags'; + +export async function findExistingPRReviewId( + repoOwner: string, + repoName: string, + pullRequestNumber: number, + benchName: string, + token: string, +) { + const client = github.getOctokit(token); + + const existingReviewsResponse = await client.rest.pulls.listReviews({ + owner: repoOwner, + repo: repoName, + // eslint-disable-next-line @typescript-eslint/naming-convention + pull_number: pullRequestNumber, + }); + + const existingReview = existingReviewsResponse.data.find((review) => + review.body.startsWith(benchmarkStartTag(benchName)), + ); + + return existingReview?.id; +} diff --git a/src/comment/leaveCommitComment.ts b/src/comment/leaveCommitComment.ts new file mode 100644 index 000000000..612e04d40 --- /dev/null +++ b/src/comment/leaveCommitComment.ts @@ -0,0 +1,22 @@ +import * as github from '@actions/github'; +import { wrapBodyWithBenchmarkTags } from './benchmarkCommentTags'; + +export async function leaveCommitComment( + repoOwner: string, + repoName: string, + commitId: string, + body: string, + commentId: string, + token: string, +) { + const client = github.getOctokit(token); + const response = await client.rest.repos.createCommitComment({ + owner: repoOwner, + repo: repoName, + // eslint-disable-next-line @typescript-eslint/naming-convention + commit_sha: commitId, + body: wrapBodyWithBenchmarkTags(commentId, body), + }); + console.log(`Comment was sent to ${response.url}. Response:`, response.status, response.data); + return response; +} diff --git a/src/comment/leavePRComment.ts b/src/comment/leavePRComment.ts new file mode 100644 index 000000000..b2dc53672 --- /dev/null +++ b/src/comment/leavePRComment.ts @@ -0,0 +1,56 @@ +import * as github from '@actions/github'; +import { wrapBodyWithBenchmarkTags } from './benchmarkCommentTags'; +import { findExistingPRReviewId } from './findExistingPRReviewId'; + +export async function leavePRComment( + repoOwner: string, + repoName: string, + pullRequestNumber: number, + body: string, + commentId: string, + token: string, +) { + const client = github.getOctokit(token); + + const bodyWithTags = wrapBodyWithBenchmarkTags(commentId, body); + + const existingCommentId = await findExistingPRReviewId(repoOwner, repoName, pullRequestNumber, commentId, token); + + if (!existingCommentId) { + const createReviewResponse = await client.rest.pulls.createReview({ + owner: repoOwner, + repo: repoName, + // eslint-disable-next-line @typescript-eslint/naming-convention + pull_number: pullRequestNumber, + event: 'COMMENT', + body: bodyWithTags, + }); + + console.log( + `Comment was created via ${createReviewResponse.url}. Response:`, + createReviewResponse.status, + createReviewResponse.data, + ); + + return createReviewResponse; + } + + const updateReviewResponse = await client.rest.pulls.updateReview({ + owner: repoOwner, + repo: repoName, + // eslint-disable-next-line @typescript-eslint/naming-convention + pull_number: pullRequestNumber, + // eslint-disable-next-line @typescript-eslint/naming-convention + review_id: existingCommentId, + event: 'COMMENT', + body: bodyWithTags, + }); + + console.log( + `Comment was updated via ${updateReviewResponse.url}. Response:`, + updateReviewResponse.status, + updateReviewResponse.data, + ); + + return updateReviewResponse; +} diff --git a/src/write.ts b/src/write.ts index 9cd04189c..133d0bc58 100644 --- a/src/write.ts +++ b/src/write.ts @@ -7,6 +7,8 @@ import * as git from './git'; import { Benchmark, BenchmarkResult } from './extract'; import { Config, ToolType } from './config'; import { DEFAULT_INDEX_HTML } from './default_index_html'; +import { leavePRComment } from './comment/leavePRComment'; +import { leaveCommitComment } from './comment/leaveCommitComment'; export type BenchmarkSuites = { [name: string]: Benchmark[] }; export interface DataJson { @@ -236,35 +238,15 @@ function buildAlertComment( return lines.join('\n'); } -async function leaveComment(commitId: string, body: string, token: string) { +async function leaveComment(commitId: string, body: string, commentId: string, token: string) { core.debug('Sending comment:\n' + body); const repoMetadata = getCurrentRepoMetadata(); - const repoUrl = repoMetadata.html_url ?? ''; const pr = github.context.payload.pull_request; - const client = github.getOctokit(token); - - const res = await (pr?.number - ? client.rest.pulls.createReview({ - owner: repoMetadata.owner.login, - repo: repoMetadata.name, - // eslint-disable-next-line @typescript-eslint/naming-convention - pull_number: pr.number, - event: 'COMMENT', - body, - }) - : client.rest.repos.createCommitComment({ - owner: repoMetadata.owner.login, - repo: repoMetadata.name, - // eslint-disable-next-line @typescript-eslint/naming-convention - commit_sha: commitId, - body, - })); - - const commitUrl = `${repoUrl}/commit/${commitId}`; - console.log(`Comment was sent to ${commitUrl}. Response:`, res.status, res.data); - - return res; + + return await (pr?.number + ? leavePRComment(repoMetadata.owner.login, repoMetadata.name, pr.number, body, commentId, token) + : leaveCommitComment(repoMetadata.owner.login, repoMetadata.name, commitId, body, commentId, token)); } async function handleComment(benchName: string, curSuite: Benchmark, prevSuite: Benchmark, config: Config) { @@ -283,7 +265,7 @@ async function handleComment(benchName: string, curSuite: Benchmark, prevSuite: const body = buildComment(benchName, curSuite, prevSuite); - await leaveComment(curSuite.commit.id, body, githubToken); + await leaveComment(curSuite.commit.id, body, `${benchName} Summary`, githubToken); } async function handleAlert(benchName: string, curSuite: Benchmark, prevSuite: Benchmark, config: Config) { @@ -309,7 +291,7 @@ async function handleAlert(benchName: string, curSuite: Benchmark, prevSuite: Be if (!githubToken) { throw new Error("'comment-on-alert' input is set but 'github-token' input is not set"); } - const res = await leaveComment(curSuite.commit.id, body, githubToken); + const res = await leaveComment(curSuite.commit.id, body, `${benchName} Alert`, githubToken); url = res.data.html_url; message = body + `\nComment was generated at ${url}`; } From 2e0dafcffee1e3a6602f2897c30445f8a7493e71 Mon Sep 17 00:00:00 2001 From: Chris Trzesniewski Date: Fri, 2 Feb 2024 14:21:51 +0100 Subject: [PATCH 09/34] enable go benchmark --- .github/workflows/ci.yml | 64 ++++++++++++++++++++-------------------- 1 file changed, 32 insertions(+), 32 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 44ebeada5..9e29ce012 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -43,38 +43,38 @@ jobs: comment-always: true - run: node ./dist/scripts/ci_validate_modification.js before_data.js 'Rust Benchmark' -# go: -# name: Run Go benchmark example -# runs-on: ubuntu-latest -# steps: -# - uses: actions/checkout@v4 -# - uses: actions/setup-node@v4 -# with: -# node-version: 20 -# cache: 'npm' -# - uses: actions/setup-go@v4 -# with: -# go-version: "stable" -# - run: npm ci -# - run: npm run build -# - name: Run benchmark -# run: cd examples/go && go test -bench 'BenchmarkFib' | tee output.txt -# - name: Save previous data.js -# run: | -# git fetch origin gh-pages -# git checkout gh-pages -# cp ./dev/bench/data.js before_data.js -# git checkout - -# - name: Store benchmark result -# uses: ./ -# with: -# name: Go Benchmark -# tool: 'go' -# output-file-path: examples/go/output.txt -# skip-fetch-gh-pages: true -# fail-on-alert: true -# summary-always: true -# - run: node ./dist/scripts/ci_validate_modification.js before_data.js 'Go Benchmark' + go: + name: Run Go benchmark example + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version: 20 + cache: 'npm' + - uses: actions/setup-go@v4 + with: + go-version: "stable" + - run: npm ci + - run: npm run build + - name: Run benchmark + run: cd examples/go && go test -bench 'BenchmarkFib' | tee output.txt + - name: Save previous data.js + run: | + git fetch origin gh-pages + git checkout gh-pages + cp ./dev/bench/data.js before_data.js + git checkout - + - name: Store benchmark result + uses: ./ + with: + name: Go Benchmark + tool: 'go' + output-file-path: examples/go/output.txt + skip-fetch-gh-pages: true + fail-on-alert: true + summary-always: true + - run: node ./dist/scripts/ci_validate_modification.js before_data.js 'Go Benchmark' # # benchmarkjs: # name: Run JavaScript benchmark example From a18867ac3d0cd4c9c628a182cc0d9b092363a851 Mon Sep 17 00:00:00 2001 From: Chris Trzesniewski Date: Fri, 2 Feb 2024 14:27:31 +0100 Subject: [PATCH 10/34] add debug comments --- src/comment/findExistingPRReviewId.ts | 3 ++ src/comment/leaveCommitComment.ts | 3 ++ src/comment/leavePRComment.ts | 73 +++++++++++++++++---------- 3 files changed, 51 insertions(+), 28 deletions(-) diff --git a/src/comment/findExistingPRReviewId.ts b/src/comment/findExistingPRReviewId.ts index 1b015e64a..b41c594fc 100644 --- a/src/comment/findExistingPRReviewId.ts +++ b/src/comment/findExistingPRReviewId.ts @@ -1,5 +1,6 @@ import * as github from '@actions/github'; import { benchmarkStartTag } from './benchmarkCommentTags'; +import * as core from '@actions/core'; export async function findExistingPRReviewId( repoOwner: string, @@ -8,6 +9,7 @@ export async function findExistingPRReviewId( benchName: string, token: string, ) { + core.debug('findExistingPRReviewId start'); const client = github.getOctokit(token); const existingReviewsResponse = await client.rest.pulls.listReviews({ @@ -21,5 +23,6 @@ export async function findExistingPRReviewId( review.body.startsWith(benchmarkStartTag(benchName)), ); + core.debug('findExistingPRReviewId start'); return existingReview?.id; } diff --git a/src/comment/leaveCommitComment.ts b/src/comment/leaveCommitComment.ts index 612e04d40..c25b67145 100644 --- a/src/comment/leaveCommitComment.ts +++ b/src/comment/leaveCommitComment.ts @@ -1,4 +1,5 @@ import * as github from '@actions/github'; +import * as core from '@actions/core'; import { wrapBodyWithBenchmarkTags } from './benchmarkCommentTags'; export async function leaveCommitComment( @@ -9,6 +10,7 @@ export async function leaveCommitComment( commentId: string, token: string, ) { + core.debug('leaveCommitComment start'); const client = github.getOctokit(token); const response = await client.rest.repos.createCommitComment({ owner: repoOwner, @@ -18,5 +20,6 @@ export async function leaveCommitComment( body: wrapBodyWithBenchmarkTags(commentId, body), }); console.log(`Comment was sent to ${response.url}. Response:`, response.status, response.data); + core.debug('leaveCommitComment end'); return response; } diff --git a/src/comment/leavePRComment.ts b/src/comment/leavePRComment.ts index b2dc53672..6f8cdbf2e 100644 --- a/src/comment/leavePRComment.ts +++ b/src/comment/leavePRComment.ts @@ -1,6 +1,7 @@ import * as github from '@actions/github'; import { wrapBodyWithBenchmarkTags } from './benchmarkCommentTags'; import { findExistingPRReviewId } from './findExistingPRReviewId'; +import * as core from '@actions/core'; export async function leavePRComment( repoOwner: string, @@ -10,47 +11,63 @@ export async function leavePRComment( commentId: string, token: string, ) { - const client = github.getOctokit(token); + try { + core.debug('leavePRComment start'); + const client = github.getOctokit(token); - const bodyWithTags = wrapBodyWithBenchmarkTags(commentId, body); + const bodyWithTags = wrapBodyWithBenchmarkTags(commentId, body); - const existingCommentId = await findExistingPRReviewId(repoOwner, repoName, pullRequestNumber, commentId, token); + const existingCommentId = await findExistingPRReviewId( + repoOwner, + repoName, + pullRequestNumber, + commentId, + token, + ); + + if (!existingCommentId) { + core.debug('creating new pr comment'); + const createReviewResponse = await client.rest.pulls.createReview({ + owner: repoOwner, + repo: repoName, + // eslint-disable-next-line @typescript-eslint/naming-convention + pull_number: pullRequestNumber, + event: 'COMMENT', + body: bodyWithTags, + }); + + console.log( + `Comment was created via ${createReviewResponse.url}. Response:`, + createReviewResponse.status, + createReviewResponse.data, + ); - if (!existingCommentId) { - const createReviewResponse = await client.rest.pulls.createReview({ + core.debug('leavePRComment end'); + return createReviewResponse; + } + + core.debug('updating existing pr comment'); + const updateReviewResponse = await client.rest.pulls.updateReview({ owner: repoOwner, repo: repoName, // eslint-disable-next-line @typescript-eslint/naming-convention pull_number: pullRequestNumber, + // eslint-disable-next-line @typescript-eslint/naming-convention + review_id: existingCommentId, event: 'COMMENT', body: bodyWithTags, }); console.log( - `Comment was created via ${createReviewResponse.url}. Response:`, - createReviewResponse.status, - createReviewResponse.data, + `Comment was updated via ${updateReviewResponse.url}. Response:`, + updateReviewResponse.status, + updateReviewResponse.data, ); + core.debug('leavePRComment end'); - return createReviewResponse; + return updateReviewResponse; + } catch (err) { + console.log('error', err); + throw err; } - - const updateReviewResponse = await client.rest.pulls.updateReview({ - owner: repoOwner, - repo: repoName, - // eslint-disable-next-line @typescript-eslint/naming-convention - pull_number: pullRequestNumber, - // eslint-disable-next-line @typescript-eslint/naming-convention - review_id: existingCommentId, - event: 'COMMENT', - body: bodyWithTags, - }); - - console.log( - `Comment was updated via ${updateReviewResponse.url}. Response:`, - updateReviewResponse.status, - updateReviewResponse.data, - ); - - return updateReviewResponse; } From 050be97aa7496a867d6f1997316c4df7a8435602 Mon Sep 17 00:00:00 2001 From: Chris Trzesniewski Date: Fri, 2 Feb 2024 14:29:01 +0100 Subject: [PATCH 11/34] fix updating comment request --- .github/workflows/ci.yml | 1 + src/comment/leavePRComment.ts | 1 - 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9e29ce012..022da8cbe 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -74,6 +74,7 @@ jobs: skip-fetch-gh-pages: true fail-on-alert: true summary-always: true + comment-always: true - run: node ./dist/scripts/ci_validate_modification.js before_data.js 'Go Benchmark' # # benchmarkjs: diff --git a/src/comment/leavePRComment.ts b/src/comment/leavePRComment.ts index 6f8cdbf2e..d9bb94f2e 100644 --- a/src/comment/leavePRComment.ts +++ b/src/comment/leavePRComment.ts @@ -54,7 +54,6 @@ export async function leavePRComment( pull_number: pullRequestNumber, // eslint-disable-next-line @typescript-eslint/naming-convention review_id: existingCommentId, - event: 'COMMENT', body: bodyWithTags, }); From 962b44f792ac59d2a7cad090438bc5b4d47d88a5 Mon Sep 17 00:00:00 2001 From: Chris Trzesniewski Date: Fri, 2 Feb 2024 14:30:34 +0100 Subject: [PATCH 12/34] add GH token to Go bench --- .github/workflows/ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 022da8cbe..5e045212b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -72,6 +72,7 @@ jobs: tool: 'go' output-file-path: examples/go/output.txt skip-fetch-gh-pages: true + github-token: ${{ secrets.GITHUB_TOKEN }} fail-on-alert: true summary-always: true comment-always: true From a43b1aaa8d54821246ec89f97002366734687943 Mon Sep 17 00:00:00 2001 From: Chris Trzesniewski Date: Fri, 2 Feb 2024 14:34:31 +0100 Subject: [PATCH 13/34] enable benchmarkjs ci run --- .github/workflows/ci.yml | 62 +++++++++++++++++++++------------------- 1 file changed, 32 insertions(+), 30 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5e045212b..8628d186f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -77,36 +77,38 @@ jobs: summary-always: true comment-always: true - run: node ./dist/scripts/ci_validate_modification.js before_data.js 'Go Benchmark' -# -# benchmarkjs: -# name: Run JavaScript benchmark example -# runs-on: ubuntu-latest -# steps: -# - uses: actions/checkout@v4 -# - uses: actions/setup-node@v4 -# with: -# node-version: 20 -# cache: 'npm' -# - run: npm ci -# - run: npm run build -# - name: Run benchmark -# run: cd examples/benchmarkjs && npm install && node bench.js | tee output.txt -# - name: Save previous data.js -# run: | -# git fetch origin gh-pages -# git checkout gh-pages -# cp ./dev/bench/data.js before_data.js -# git checkout - -# - name: Store benchmark result -# uses: ./ -# with: -# name: Benchmark.js Benchmark -# tool: 'benchmarkjs' -# output-file-path: examples/benchmarkjs/output.txt -# skip-fetch-gh-pages: true -# fail-on-alert: true -# summary-always: true -# - run: node ./dist/scripts/ci_validate_modification.js before_data.js 'Benchmark.js Benchmark' + + benchmarkjs: + name: Run JavaScript benchmark example + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version: 20 + cache: 'npm' + - run: npm ci + - run: npm run build + - name: Run benchmark + run: cd examples/benchmarkjs && npm install && node bench.js | tee output.txt + - name: Save previous data.js + run: | + git fetch origin gh-pages + git checkout gh-pages + cp ./dev/bench/data.js before_data.js + git checkout - + - name: Store benchmark result + uses: ./ + with: + name: Benchmark.js Benchmark + tool: 'benchmarkjs' + output-file-path: examples/benchmarkjs/output.txt + skip-fetch-gh-pages: true + github-token: ${{ secrets.GITHUB_TOKEN }} + fail-on-alert: true + summary-always: true + comment-always: true + - run: node ./dist/scripts/ci_validate_modification.js before_data.js 'Benchmark.js Benchmark' # # pytest-benchmark: # name: Run Pytest benchmark example From d0dea1e57c781153e692bbb102322178b6c82b0a Mon Sep 17 00:00:00 2001 From: Chris Trzesniewski Date: Fri, 2 Feb 2024 14:38:38 +0100 Subject: [PATCH 14/34] enable remaining ci steps --- .github/workflows/ci.yml | 572 ++++++++++++++++++++------------------- 1 file changed, 292 insertions(+), 280 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8628d186f..bd8933df7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -109,283 +109,295 @@ jobs: summary-always: true comment-always: true - run: node ./dist/scripts/ci_validate_modification.js before_data.js 'Benchmark.js Benchmark' -# -# pytest-benchmark: -# name: Run Pytest benchmark example -# runs-on: ubuntu-latest -# steps: -# - uses: actions/checkout@v4 -# - uses: actions/setup-node@v4 -# with: -# node-version: 20 -# cache: 'npm' -# - uses: actions/setup-python@v5 -# with: -# python-version: 3.9 -# - run: npm ci -# - run: npm run build -# - name: Run benchmark -# run: | -# cd examples/pytest -# pip install -r requirements.txt -# pytest bench.py --benchmark-json output.json -# - name: Save previous data.js -# run: | -# git fetch origin gh-pages -# git checkout gh-pages -# cp ./dev/bench/data.js before_data.js -# git checkout - -# - name: Store benchmark result -# uses: ./ -# with: -# name: Python Benchmark with pytest-benchmark -# tool: 'pytest' -# output-file-path: examples/pytest/output.json -# skip-fetch-gh-pages: true -# fail-on-alert: true -# summary-always: true -# - run: node ./dist/scripts/ci_validate_modification.js before_data.js 'Python Benchmark with pytest-benchmark' -# -# google-benchmark-framework: -# name: Run Google C++ Benchmark Framework example -# runs-on: ubuntu-20.04 -# steps: -# - uses: actions/checkout@v4 -# - uses: actions/setup-node@v4 -# with: -# node-version: 20 -# cache: 'npm' -# - run: npm ci -# - run: npm run build -# - name: Cache Benchmark library -# uses: actions/cache@v4 -# with: -# path: examples/cpp/benchmark -# key: ${{ runner.os }}-googlebenchmark-v1.5.0 -# - name: Run benchmark -# run: | -# cd examples/cpp -# make json -# - name: Save previous data.js -# run: | -# git fetch origin gh-pages -# git checkout gh-pages -# cp ./dev/bench/data.js before_data.js -# git checkout - -# - name: Store benchmark result -# uses: ./ -# with: -# name: C++ Benchmark -# tool: 'googlecpp' -# output-file-path: examples/cpp/benchmark_result.json -# skip-fetch-gh-pages: true -# fail-on-alert: true -# summary-always: true -# - run: node ./dist/scripts/ci_validate_modification.js before_data.js 'C++ Benchmark' -# -# catch2-framework: -# name: Run Catch2 C++ Benchmark Framework example -# runs-on: ubuntu-20.04 -# steps: -# - uses: actions/checkout@v4 -# - uses: actions/setup-node@v4 -# with: -# node-version: 20 -# cache: 'npm' -# - run: npm ci -# - run: npm run build -# - name: Run benchmark -# run: | -# cd examples/catch2 -# mkdir build && cd build -# cmake -DCMAKE_BUILD_TYPE=Release .. -# cmake --build . --config Release -# ./Catch2_bench > ../benchmark_result.txt -# - name: Save previous data.js -# run: | -# git fetch origin gh-pages -# git checkout gh-pages -# cp ./dev/bench/data.js before_data.js -# git checkout - -# - name: Store benchmark result -# uses: ./ -# with: -# name: Catch2 Benchmark -# tool: 'catch2' -# output-file-path: examples/catch2/benchmark_result.txt -# skip-fetch-gh-pages: true -# fail-on-alert: true -# summary-always: true -# - run: node ./dist/scripts/ci_validate_modification.js before_data.js 'Catch2 Benchmark' -# -# julia-benchmark: -# name: Run Julia benchmark example -# runs-on: ubuntu-latest -# steps: -# - uses: actions/checkout@v4 -# - uses: actions/setup-node@v4 -# with: -# node-version: 20 -# cache: 'npm' -# - run: npm ci -# - run: npm run build -# - uses: julia-actions/setup-julia@v1 -# with: -# version: '1' -# arch: x64 -# - name: Run benchmark -# run: | -# cd examples/julia -# julia --project --color=yes -e ' -# using Pkg; -# Pkg.instantiate(); -# include("fib.jl")' -# - name: Save previous data.js -# run: | -# git fetch origin gh-pages -# git checkout gh-pages -# cp ./dev/bench/data.js before_data.js -# git checkout - -# - name: Store benchmark result -# uses: ./ -# with: -# name: Julia benchmark -# tool: 'julia' -# output-file-path: examples/julia/output.json -# skip-fetch-gh-pages: true -# fail-on-alert: true -# summary-always: true -# - run: node ./dist/scripts/ci_validate_modification.js before_data.js 'Julia benchmark' -# -# benchmarkdotnet-framework: -# name: Run Benchmark.Net .Net Benchmark Framework example -# runs-on: ubuntu-latest -# steps: -# - uses: actions/checkout@v4 -# - uses: actions/setup-node@v4 -# with: -# node-version: 20 -# cache: 'npm' -# - uses: actions/setup-dotnet@v3 -# with: -# dotnet-version: '6.0.101' # SDK Version to use. keep in line with examples/benchmarkdotnet/global.json -# - run: npm ci -# - run: npm run build -# - name: Run benchmark -# run: cd examples/benchmarkdotnet && dotnet run --exporters json --filter '*' -# - name: Save previous data.js -# run: | -# git fetch origin gh-pages -# git checkout gh-pages -# cp ./dev/bench/data.js before_data.js -# git checkout - -# - name: Store benchmark result -# uses: ./ -# with: -# name: Benchmark.Net Benchmark -# tool: 'benchmarkdotnet' -# output-file-path: examples/benchmarkdotnet/BenchmarkDotNet.Artifacts/results/Sample.Benchmarks-report-full-compressed.json -# skip-fetch-gh-pages: true -# fail-on-alert: true -# summary-always: true -# - run: node ./dist/scripts/ci_validate_modification.js before_data.js 'Benchmark.Net Benchmark' -# -# jmh: -# name: Run JMH Java Benchmark Framework example -# runs-on: ubuntu-latest -# steps: -# - uses: actions/checkout@v4 -# - uses: actions/setup-node@v4 -# with: -# node-version: 20 -# cache: 'npm' -# - uses: actions/setup-java@v4 -# with: -# distribution: 'adopt' -# java-version: '11' -# - run: npm ci -# - run: npm run build -# - name: Run benchmark -# run: | -# cd examples/java -# mvn clean verify -# java -jar target/benchmarks.jar -wi 1 -i 3 -f 1 -rf json -# - name: Save previous data.js -# run: | -# git fetch origin gh-pages -# git checkout gh-pages -# cp ./dev/bench/data.js before_data.js -# git checkout - -# - name: Store benchmark result -# uses: ./ -# with: -# name: JMH Benchmark -# tool: "jmh" -# output-file-path: examples/java/jmh-result.json -# skip-fetch-gh-pages: true -# fail-on-alert: true -# summary-always: true -# - run: node ./dist/scripts/ci_validate_modification.js before_data.js 'JMH Benchmark' -# -# only-alert-with-cache: -# name: Run alert check with actions/cache -# runs-on: ubuntu-latest -# steps: -# - uses: actions/checkout@v4 -# - uses: actions/setup-node@v4 -# with: -# node-version: 20 -# cache: 'npm' -# - run: npm ci -# - run: npm run build -# - uses: actions/setup-go@v4 -# with: -# go-version: "stable" -# - name: Run benchmark -# run: cd examples/go && go test -bench 'BenchmarkFib' | tee output.txt -# - name: Download previous benchmark data -# uses: actions/cache@v4 -# with: -# path: ./cache -# key: ${{ runner.os }}-ci-cache-test -# - name: Store benchmark result -# uses: ./ -# with: -# name: Alert setup example with cache -# tool: 'go' -# output-file-path: examples/go/output.txt -# external-data-json-path: ./cache/benchmark-data.json -# github-token: ${{ secrets.GITHUB_TOKEN }} -# alert-threshold: '200%' -# comment-on-alert: true -# fail-on-alert: true -# alert-comment-cc-users: '@ktrz' -# unit-tests: -# name: Run unit tests -# runs-on: ubuntu-latest -# steps: -# - uses: actions/checkout@v4 -# - uses: actions/setup-node@v4 -# with: -# node-version: 20 -# cache: 'npm' -# - run: npm ci -# - run: npm run coverage -# - name: Apply yamllint -# run: | -# sudo pip install yamllint -# yamllint --strict .github/workflows -# - uses: codecov/codecov-action@v2 -# -# lint: -# name: Run linting and formatting check -# runs-on: ubuntu-latest -# steps: -# - uses: actions/checkout@v4 -# - uses: actions/setup-node@v4 -# with: -# node-version: 20 -# cache: 'npm' -# - run: npm ci -# - run: npm run lint -# - run: npm run format:check + + pytest-benchmark: + name: Run Pytest benchmark example + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version: 20 + cache: 'npm' + - uses: actions/setup-python@v5 + with: + python-version: 3.9 + - run: npm ci + - run: npm run build + - name: Run benchmark + run: | + cd examples/pytest + pip install -r requirements.txt + pytest bench.py --benchmark-json output.json + - name: Save previous data.js + run: | + git fetch origin gh-pages + git checkout gh-pages + cp ./dev/bench/data.js before_data.js + git checkout - + - name: Store benchmark result + uses: ./ + with: + name: Python Benchmark with pytest-benchmark + tool: 'pytest' + output-file-path: examples/pytest/output.json + github-token: ${{ secrets.GITHUB_TOKEN }} + skip-fetch-gh-pages: true + fail-on-alert: true + summary-always: true + comment-always: true + - run: node ./dist/scripts/ci_validate_modification.js before_data.js 'Python Benchmark with pytest-benchmark' + + google-benchmark-framework: + name: Run Google C++ Benchmark Framework example + runs-on: ubuntu-20.04 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version: 20 + cache: 'npm' + - run: npm ci + - run: npm run build + - name: Cache Benchmark library + uses: actions/cache@v4 + with: + path: examples/cpp/benchmark + key: ${{ runner.os }}-googlebenchmark-v1.5.0 + - name: Run benchmark + run: | + cd examples/cpp + make json + - name: Save previous data.js + run: | + git fetch origin gh-pages + git checkout gh-pages + cp ./dev/bench/data.js before_data.js + git checkout - + - name: Store benchmark result + uses: ./ + with: + name: C++ Benchmark + tool: 'googlecpp' + output-file-path: examples/cpp/benchmark_result.json + github-token: ${{ secrets.GITHUB_TOKEN }} + skip-fetch-gh-pages: true + fail-on-alert: true + summary-always: true + comment-always: true + - run: node ./dist/scripts/ci_validate_modification.js before_data.js 'C++ Benchmark' + + catch2-framework: + name: Run Catch2 C++ Benchmark Framework example + runs-on: ubuntu-20.04 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version: 20 + cache: 'npm' + - run: npm ci + - run: npm run build + - name: Run benchmark + run: | + cd examples/catch2 + mkdir build && cd build + cmake -DCMAKE_BUILD_TYPE=Release .. + cmake --build . --config Release + ./Catch2_bench > ../benchmark_result.txt + - name: Save previous data.js + run: | + git fetch origin gh-pages + git checkout gh-pages + cp ./dev/bench/data.js before_data.js + git checkout - + - name: Store benchmark result + uses: ./ + with: + name: Catch2 Benchmark + tool: 'catch2' + output-file-path: examples/catch2/benchmark_result.txt + github-token: ${{ secrets.GITHUB_TOKEN }} + skip-fetch-gh-pages: true + fail-on-alert: true + summary-always: true + comment-always: true + - run: node ./dist/scripts/ci_validate_modification.js before_data.js 'Catch2 Benchmark' + + julia-benchmark: + name: Run Julia benchmark example + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version: 20 + cache: 'npm' + - run: npm ci + - run: npm run build + - uses: julia-actions/setup-julia@v1 + with: + version: '1' + arch: x64 + - name: Run benchmark + run: | + cd examples/julia + julia --project --color=yes -e ' + using Pkg; + Pkg.instantiate(); + include("fib.jl")' + - name: Save previous data.js + run: | + git fetch origin gh-pages + git checkout gh-pages + cp ./dev/bench/data.js before_data.js + git checkout - + - name: Store benchmark result + uses: ./ + with: + name: Julia benchmark + tool: 'julia' + output-file-path: examples/julia/output.json + github-token: ${{ secrets.GITHUB_TOKEN }} + skip-fetch-gh-pages: true + fail-on-alert: true + summary-always: true + comment-always: true + - run: node ./dist/scripts/ci_validate_modification.js before_data.js 'Julia benchmark' + + benchmarkdotnet-framework: + name: Run Benchmark.Net .Net Benchmark Framework example + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version: 20 + cache: 'npm' + - uses: actions/setup-dotnet@v3 + with: + dotnet-version: '6.0.101' # SDK Version to use. keep in line with examples/benchmarkdotnet/global.json + - run: npm ci + - run: npm run build + - name: Run benchmark + run: cd examples/benchmarkdotnet && dotnet run --exporters json --filter '*' + - name: Save previous data.js + run: | + git fetch origin gh-pages + git checkout gh-pages + cp ./dev/bench/data.js before_data.js + git checkout - + - name: Store benchmark result + uses: ./ + with: + name: Benchmark.Net Benchmark + tool: 'benchmarkdotnet' + output-file-path: examples/benchmarkdotnet/BenchmarkDotNet.Artifacts/results/Sample.Benchmarks-report-full-compressed.json + github-token: ${{ secrets.GITHUB_TOKEN }} + skip-fetch-gh-pages: true + fail-on-alert: true + summary-always: true + comment-always: true + - run: node ./dist/scripts/ci_validate_modification.js before_data.js 'Benchmark.Net Benchmark' + + jmh: + name: Run JMH Java Benchmark Framework example + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version: 20 + cache: 'npm' + - uses: actions/setup-java@v4 + with: + distribution: 'adopt' + java-version: '11' + - run: npm ci + - run: npm run build + - name: Run benchmark + run: | + cd examples/java + mvn clean verify + java -jar target/benchmarks.jar -wi 1 -i 3 -f 1 -rf json + - name: Save previous data.js + run: | + git fetch origin gh-pages + git checkout gh-pages + cp ./dev/bench/data.js before_data.js + git checkout - + - name: Store benchmark result + uses: ./ + with: + name: JMH Benchmark + tool: "jmh" + output-file-path: examples/java/jmh-result.json + skip-fetch-gh-pages: true + github-token: ${{ secrets.GITHUB_TOKEN }} + fail-on-alert: true + summary-always: true + comment-always: true + - run: node ./dist/scripts/ci_validate_modification.js before_data.js 'JMH Benchmark' + + only-alert-with-cache: + name: Run alert check with actions/cache + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version: 20 + cache: 'npm' + - run: npm ci + - run: npm run build + - uses: actions/setup-go@v4 + with: + go-version: "stable" + - name: Run benchmark + run: cd examples/go && go test -bench 'BenchmarkFib' | tee output.txt + - name: Download previous benchmark data + uses: actions/cache@v4 + with: + path: ./cache + key: ${{ runner.os }}-ci-cache-test + - name: Store benchmark result + uses: ./ + with: + name: Alert setup example with cache + tool: 'go' + output-file-path: examples/go/output.txt + external-data-json-path: ./cache/benchmark-data.json + github-token: ${{ secrets.GITHUB_TOKEN }} + alert-threshold: '200%' + comment-on-alert: true + fail-on-alert: true + alert-comment-cc-users: '@ktrz' + unit-tests: + name: Run unit tests + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version: 20 + cache: 'npm' + - run: npm ci + - run: npm run coverage + - name: Apply yamllint + run: | + sudo pip install yamllint + yamllint --strict .github/workflows + - uses: codecov/codecov-action@v2 + + lint: + name: Run linting and formatting check + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version: 20 + cache: 'npm' + - run: npm ci + - run: npm run lint + - run: npm run format:check From 1f4122e4294b7ebceffcf3de3b5194de8cdddddd Mon Sep 17 00:00:00 2001 From: Chris Trzesniewski Date: Wed, 27 Mar 2024 09:57:26 +0100 Subject: [PATCH 15/34] fix tests --- test/write.spec.ts | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/test/write.spec.ts b/test/write.spec.ts index fee85a224..121ce517b 100644 --- a/test/write.spec.ts +++ b/test/write.spec.ts @@ -8,6 +8,7 @@ import { Benchmark } from '../src/extract'; import { DataJson, writeBenchmark } from '../src/write'; import { expect } from '@jest/globals'; import { FakedOctokit, fakedRepos } from './fakedOctokit'; +import { wrapBodyWithBenchmarkTags } from '../src/comment/benchmarkCommentTags'; const ok: (x: any, msg?: string) => asserts x = (x, msg) => { try { @@ -771,7 +772,10 @@ describe.each(['https://github.com', 'https://github.enterprise.corp'])('writeBe // Last line is appended only for failure message const messageLines = caughtError.message.split('\n'); ok(messageLines.length > 0); - const expectedMessage = messageLines.slice(0, -1).join('\n'); + const expectedMessage = wrapBodyWithBenchmarkTags( + 'Test benchmark Alert', + messageLines.slice(0, -1).join('\n'), + ); ok( fakedRepos.spyOpts.length > 0, `len: ${fakedRepos.spyOpts.length}, caught: ${caughtError.message}`, From 65bdfe156ae43299f55a3ba2948447efabdc4575 Mon Sep 17 00:00:00 2001 From: Chris Trzesniewski Date: Wed, 27 Mar 2024 10:03:45 +0100 Subject: [PATCH 16/34] debug --- src/git.ts | 1 + src/write.ts | 1 + 2 files changed, 2 insertions(+) diff --git a/src/git.ts b/src/git.ts index 1a228ac6d..c84db5256 100644 --- a/src/git.ts +++ b/src/git.ts @@ -47,6 +47,7 @@ export function getServerUrlObj(repositoryUrl: string | undefined): URL { } export function getServerUrl(repositoryUrl: string | undefined): string { + core.debug(`getServerUrl ${repositoryUrl}`); return getServerUrlObj(repositoryUrl).origin; } diff --git a/src/write.ts b/src/write.ts index 133d0bc58..74eb3af11 100644 --- a/src/write.ts +++ b/src/write.ts @@ -120,6 +120,7 @@ function findAlerts(curSuite: Benchmark, prevSuite: Benchmark, threshold: number } function getCurrentRepoMetadata() { + core.debug('getCurrentRepoMetadata'); const { repo, owner } = github.context.repo; const serverUrl = git.getServerUrl(github.context.payload.repository?.html_url); return { From f8f4cd3177e9d71447a7391ab06c1119b0845b30 Mon Sep 17 00:00:00 2001 From: Chris Trzesniewski Date: Wed, 27 Mar 2024 10:10:31 +0100 Subject: [PATCH 17/34] add try catch --- src/write.ts | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/src/write.ts b/src/write.ts index 74eb3af11..02e349175 100644 --- a/src/write.ts +++ b/src/write.ts @@ -240,14 +240,19 @@ function buildAlertComment( } async function leaveComment(commitId: string, body: string, commentId: string, token: string) { - core.debug('Sending comment:\n' + body); + try { + core.debug('Sending comment:\n' + body); - const repoMetadata = getCurrentRepoMetadata(); - const pr = github.context.payload.pull_request; + const repoMetadata = getCurrentRepoMetadata(); + const pr = github.context.payload.pull_request; - return await (pr?.number - ? leavePRComment(repoMetadata.owner.login, repoMetadata.name, pr.number, body, commentId, token) - : leaveCommitComment(repoMetadata.owner.login, repoMetadata.name, commitId, body, commentId, token)); + return await (pr?.number + ? leavePRComment(repoMetadata.owner.login, repoMetadata.name, pr.number, body, commentId, token) + : leaveCommitComment(repoMetadata.owner.login, repoMetadata.name, commitId, body, commentId, token)); + } catch (e) { + core.warning(`error: ${e}`); + return; + } } async function handleComment(benchName: string, curSuite: Benchmark, prevSuite: Benchmark, config: Config) { From d231b43a7a2934c71acfbbd15830e159624e41d7 Mon Sep 17 00:00:00 2001 From: Chris Trzesniewski Date: Wed, 27 Mar 2024 10:11:58 +0100 Subject: [PATCH 18/34] fix build err --- src/write.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/write.ts b/src/write.ts index 02e349175..0315a7592 100644 --- a/src/write.ts +++ b/src/write.ts @@ -298,7 +298,7 @@ async function handleAlert(benchName: string, curSuite: Benchmark, prevSuite: Be throw new Error("'comment-on-alert' input is set but 'github-token' input is not set"); } const res = await leaveComment(curSuite.commit.id, body, `${benchName} Alert`, githubToken); - url = res.data.html_url; + url = res?.data.html_url; message = body + `\nComment was generated at ${url}`; } From 75d81ca6c2da28f2da5b9d38f535b7f57cacaf9a Mon Sep 17 00:00:00 2001 From: Chris Trzesniewski Date: Wed, 27 Mar 2024 10:16:10 +0100 Subject: [PATCH 19/34] debug --- src/write.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/src/write.ts b/src/write.ts index 0315a7592..abfe94ddd 100644 --- a/src/write.ts +++ b/src/write.ts @@ -242,6 +242,7 @@ function buildAlertComment( async function leaveComment(commitId: string, body: string, commentId: string, token: string) { try { core.debug('Sending comment:\n' + body); + core.debug('Boo'); const repoMetadata = getCurrentRepoMetadata(); const pr = github.context.payload.pull_request; From ee3d6fb660c4fbe6afcc39e5c9b3033045f576fc Mon Sep 17 00:00:00 2001 From: Chris Trzesniewski Date: Wed, 27 Mar 2024 10:21:08 +0100 Subject: [PATCH 20/34] debug --- src/git.ts | 2 +- src/write.ts | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/git.ts b/src/git.ts index c84db5256..627ada4ee 100644 --- a/src/git.ts +++ b/src/git.ts @@ -47,7 +47,7 @@ export function getServerUrlObj(repositoryUrl: string | undefined): URL { } export function getServerUrl(repositoryUrl: string | undefined): string { - core.debug(`getServerUrl ${repositoryUrl}`); + core.warning(`getServerUrl ${repositoryUrl}`); return getServerUrlObj(repositoryUrl).origin; } diff --git a/src/write.ts b/src/write.ts index abfe94ddd..662f125f8 100644 --- a/src/write.ts +++ b/src/write.ts @@ -120,7 +120,7 @@ function findAlerts(curSuite: Benchmark, prevSuite: Benchmark, threshold: number } function getCurrentRepoMetadata() { - core.debug('getCurrentRepoMetadata'); + core.warning('getCurrentRepoMetadata'); const { repo, owner } = github.context.repo; const serverUrl = git.getServerUrl(github.context.payload.repository?.html_url); return { @@ -242,7 +242,7 @@ function buildAlertComment( async function leaveComment(commitId: string, body: string, commentId: string, token: string) { try { core.debug('Sending comment:\n' + body); - core.debug('Boo'); + core.warning('Boo'); const repoMetadata = getCurrentRepoMetadata(); const pr = github.context.payload.pull_request; From 24e5892ff1c7dec2c0be4292f52f14afc3b350ca Mon Sep 17 00:00:00 2001 From: Chris Trzesniewski Date: Wed, 27 Mar 2024 10:27:07 +0100 Subject: [PATCH 21/34] debug --- src/git.ts | 1 + src/write.ts | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/src/git.ts b/src/git.ts index 627ada4ee..df47a3add 100644 --- a/src/git.ts +++ b/src/git.ts @@ -39,6 +39,7 @@ async function capture(cmd: string, args: string[]): Promise { } export function getServerUrlObj(repositoryUrl: string | undefined): URL { + core.warning(`getServerUrlObj ${repositoryUrl}`); const urlValue = repositoryUrl && repositoryUrl.trim().length > 0 ? repositoryUrl diff --git a/src/write.ts b/src/write.ts index 662f125f8..b41c262a4 100644 --- a/src/write.ts +++ b/src/write.ts @@ -241,7 +241,7 @@ function buildAlertComment( async function leaveComment(commitId: string, body: string, commentId: string, token: string) { try { - core.debug('Sending comment:\n' + body); + core.warning('Sending comment:\n' + body); core.warning('Boo'); const repoMetadata = getCurrentRepoMetadata(); From 41f775ec073ce691987d15ba4030cf1ba94b4dc9 Mon Sep 17 00:00:00 2001 From: Chris Trzesniewski Date: Wed, 27 Mar 2024 10:31:27 +0100 Subject: [PATCH 22/34] debug --- .github/workflows/ci-results-repo.yml | 2 +- .github/workflows/ci.yml | 780 +++++++++++++------------- 2 files changed, 391 insertions(+), 391 deletions(-) diff --git a/.github/workflows/ci-results-repo.yml b/.github/workflows/ci-results-repo.yml index 4cb507ece..ba87173f4 100644 --- a/.github/workflows/ci-results-repo.yml +++ b/.github/workflows/ci-results-repo.yml @@ -378,7 +378,7 @@ jobs: cp ./dist/other-repo/dev/bench/data.js before_data.js - name: Store benchmark result - uses: benchmark-action/github-action-benchmark@v1 + uses: ./ with: name: Criterion.rs Benchmark tool: 'cargo' diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 37d16e1b7..c025d5913 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -11,333 +11,333 @@ permissions: pull-requests: write jobs: - benchmarkdotnet-framework: - name: Run Benchmark.Net .Net Benchmark Framework example - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-node@v4 - with: - node-version: 20 - cache: 'npm' - - uses: actions/setup-dotnet@v3 - with: - dotnet-version: '6.0.101' # SDK Version to use. keep in line with examples/benchmarkdotnet/global.json - - run: npm ci - - run: npm run build - - name: Run benchmark - run: cd examples/benchmarkdotnet && dotnet run --exporters json --filter '*' - - name: Save previous data.js - run: | - git fetch origin gh-pages - git checkout gh-pages - cp ./dev/bench/data.js before_data.js - git checkout - - - name: Store benchmark result - uses: ./ - with: - name: Benchmark.Net Benchmark - tool: 'benchmarkdotnet' - output-file-path: examples/benchmarkdotnet/BenchmarkDotNet.Artifacts/results/Sample.Benchmarks-report-full-compressed.json - skip-fetch-gh-pages: true - fail-on-alert: true - summary-always: true - github-token: ${{ secrets.GITHUB_TOKEN }} - comment-always: true - - run: node ./dist/scripts/ci_validate_modification.js before_data.js 'Benchmark.Net Benchmark' - - benchmarkjs: - name: Run JavaScript benchmark example - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-node@v4 - with: - node-version: 20 - cache: 'npm' - - run: npm ci - - run: npm run build - - name: Run benchmark - run: cd examples/benchmarkjs && npm install && node bench.js | tee output.txt - - name: Save previous data.js - run: | - git fetch origin gh-pages - git checkout gh-pages - cp ./dev/bench/data.js before_data.js - git checkout - - - name: Store benchmark result - uses: ./ - with: - name: Benchmark.js Benchmark - tool: 'benchmarkjs' - output-file-path: examples/benchmarkjs/output.txt - skip-fetch-gh-pages: true - fail-on-alert: true - summary-always: true - github-token: ${{ secrets.GITHUB_TOKEN }} - comment-always: true - - run: node ./dist/scripts/ci_validate_modification.js before_data.js 'Benchmark.js Benchmark' - - catch2-framework: - name: Run Catch2 C++ Benchmark Framework example - runs-on: ubuntu-20.04 - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-node@v4 - with: - node-version: 20 - cache: 'npm' - - run: npm ci - - run: npm run build - - name: Run benchmark - run: | - cd examples/catch2 - mkdir build && cd build - cmake -DCMAKE_BUILD_TYPE=Release .. - cmake --build . --config Release - ./Catch2_bench > ../benchmark_result.txt - - name: Save previous data.js - run: | - git fetch origin gh-pages - git checkout gh-pages - cp ./dev/bench/data.js before_data.js - git checkout - - - name: Store benchmark result - uses: ./ - with: - name: Catch2 Benchmark - tool: 'catch2' - output-file-path: examples/catch2/benchmark_result.txt - skip-fetch-gh-pages: true - fail-on-alert: true - summary-always: true - github-token: ${{ secrets.GITHUB_TOKEN }} - comment-always: true - - run: node ./dist/scripts/ci_validate_modification.js before_data.js 'Catch2 Benchmark' - - cpp-framework: - name: Run Google C++ Benchmark Framework example - runs-on: ubuntu-20.04 - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-node@v4 - with: - node-version: 20 - cache: 'npm' - - run: npm ci - - run: npm run build - - name: Cache Benchmark library - uses: actions/cache@v4 - with: - path: examples/cpp/benchmark - key: ${{ runner.os }}-googlebenchmark-v1.5.0 - - name: Run benchmark - run: | - cd examples/cpp - make json - - name: Save previous data.js - run: | - git fetch origin gh-pages - git checkout gh-pages - cp ./dev/bench/data.js before_data.js - git checkout - - - name: Store benchmark result - uses: ./ - with: - name: C++ Benchmark - tool: 'googlecpp' - output-file-path: examples/cpp/benchmark_result.json - skip-fetch-gh-pages: true - fail-on-alert: true - summary-always: true - github-token: ${{ secrets.GITHUB_TOKEN }} - comment-always: true - - run: node ./dist/scripts/ci_validate_modification.js before_data.js 'C++ Benchmark' - - go: - name: Run Go benchmark example - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-node@v4 - with: - node-version: 20 - cache: 'npm' - - uses: actions/setup-go@v4 - with: - go-version: "stable" - - run: npm ci - - run: npm run build - - name: Run benchmark - run: cd examples/go && go test -bench 'BenchmarkFib' | tee output.txt - - name: Save previous data.js - run: | - git fetch origin gh-pages - git checkout gh-pages - cp ./dev/bench/data.js before_data.js - git checkout - - - name: Store benchmark result - uses: ./ - with: - name: Go Benchmark - tool: 'go' - output-file-path: examples/go/output.txt - skip-fetch-gh-pages: true - fail-on-alert: true - summary-always: true - github-token: ${{ secrets.GITHUB_TOKEN }} - comment-always: true - - run: node ./dist/scripts/ci_validate_modification.js before_data.js 'Go Benchmark' - - java-jmh: - name: Run JMH Java Benchmark Framework example - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-node@v4 - with: - node-version: 20 - cache: 'npm' - - uses: actions/setup-java@v4 - with: - distribution: 'adopt' - java-version: '11' - - run: npm ci - - run: npm run build - - name: Run benchmark - run: | - cd examples/java - mvn clean verify - java -jar target/benchmarks.jar -wi 1 -i 3 -f 1 -rf json - - name: Save previous data.js - run: | - git fetch origin gh-pages - git checkout gh-pages - cp ./dev/bench/data.js before_data.js - git checkout - - - name: Store benchmark result - uses: ./ - with: - name: JMH Benchmark - tool: "jmh" - output-file-path: examples/java/jmh-result.json - skip-fetch-gh-pages: true - fail-on-alert: true - summary-always: true - github-token: ${{ secrets.GITHUB_TOKEN }} - comment-always: true - - run: node ./dist/scripts/ci_validate_modification.js before_data.js 'JMH Benchmark' - - julia-benchmark: - name: Run Julia benchmark example - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-node@v4 - with: - node-version: 20 - cache: 'npm' - - run: npm ci - - run: npm run build - - uses: julia-actions/setup-julia@v1 - with: - version: '1' - arch: x64 - - name: Run benchmark - run: | - cd examples/julia - julia --project --color=yes -e ' - using Pkg; - Pkg.instantiate(); - include("fib.jl")' - - name: Save previous data.js - run: | - git fetch origin gh-pages - git checkout gh-pages - cp ./dev/bench/data.js before_data.js - git checkout - - - name: Store benchmark result - uses: ./ - with: - name: Julia benchmark - tool: 'julia' - output-file-path: examples/julia/output.json - skip-fetch-gh-pages: true - fail-on-alert: true - summary-always: true - github-token: ${{ secrets.GITHUB_TOKEN }} - comment-always: true - - run: node ./dist/scripts/ci_validate_modification.js before_data.js 'Julia benchmark' - - pytest-benchmark: - name: Run Pytest benchmark example - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-node@v4 - with: - node-version: 20 - cache: 'npm' - - uses: actions/setup-python@v5 - with: - python-version: 3.9 - - run: npm ci - - run: npm run build - - name: Run benchmark - run: | - cd examples/pytest - pip install -r requirements.txt - pytest bench.py --benchmark-json output.json - - name: Save previous data.js - run: | - git fetch origin gh-pages - git checkout gh-pages - cp ./dev/bench/data.js before_data.js - git checkout - - - name: Store benchmark result - uses: ./ - with: - name: Python Benchmark with pytest-benchmark - tool: 'pytest' - output-file-path: examples/pytest/output.json - skip-fetch-gh-pages: true - fail-on-alert: true - summary-always: true - github-token: ${{ secrets.GITHUB_TOKEN }} - comment-always: true - - run: node ./dist/scripts/ci_validate_modification.js before_data.js 'Python Benchmark with pytest-benchmark' - - rust: - name: Run Rust benchmark example - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-node@v4 - with: - node-version: 20 - cache: 'npm' - - run: npm ci - - run: npm run build - - run: rustup toolchain update nightly && rustup default nightly - - name: Run benchmark - run: cd examples/rust && cargo +nightly bench | tee output.txt - - name: Save previous data.js - run: | - git fetch origin gh-pages - git checkout gh-pages - cp ./dev/bench/data.js before_data.js - git checkout - - - name: Store benchmark result - uses: ./ - with: - name: Rust Benchmark - tool: 'cargo' - output-file-path: examples/rust/output.txt - fail-on-alert: true - summary-always: true - github-token: ${{ secrets.GITHUB_TOKEN }} - comment-always: true - - run: node ./dist/scripts/ci_validate_modification.js before_data.js 'Rust Benchmark' +# benchmarkdotnet-framework: +# name: Run Benchmark.Net .Net Benchmark Framework example +# runs-on: ubuntu-latest +# steps: +# - uses: actions/checkout@v4 +# - uses: actions/setup-node@v4 +# with: +# node-version: 20 +# cache: 'npm' +# - uses: actions/setup-dotnet@v3 +# with: +# dotnet-version: '6.0.101' # SDK Version to use. keep in line with examples/benchmarkdotnet/global.json +# - run: npm ci +# - run: npm run build +# - name: Run benchmark +# run: cd examples/benchmarkdotnet && dotnet run --exporters json --filter '*' +# - name: Save previous data.js +# run: | +# git fetch origin gh-pages +# git checkout gh-pages +# cp ./dev/bench/data.js before_data.js +# git checkout - +# - name: Store benchmark result +# uses: ./ +# with: +# name: Benchmark.Net Benchmark +# tool: 'benchmarkdotnet' +# output-file-path: examples/benchmarkdotnet/BenchmarkDotNet.Artifacts/results/Sample.Benchmarks-report-full-compressed.json +# skip-fetch-gh-pages: true +# fail-on-alert: true +# summary-always: true +# github-token: ${{ secrets.GITHUB_TOKEN }} +# comment-always: true +# - run: node ./dist/scripts/ci_validate_modification.js before_data.js 'Benchmark.Net Benchmark' +# +# benchmarkjs: +# name: Run JavaScript benchmark example +# runs-on: ubuntu-latest +# steps: +# - uses: actions/checkout@v4 +# - uses: actions/setup-node@v4 +# with: +# node-version: 20 +# cache: 'npm' +# - run: npm ci +# - run: npm run build +# - name: Run benchmark +# run: cd examples/benchmarkjs && npm install && node bench.js | tee output.txt +# - name: Save previous data.js +# run: | +# git fetch origin gh-pages +# git checkout gh-pages +# cp ./dev/bench/data.js before_data.js +# git checkout - +# - name: Store benchmark result +# uses: ./ +# with: +# name: Benchmark.js Benchmark +# tool: 'benchmarkjs' +# output-file-path: examples/benchmarkjs/output.txt +# skip-fetch-gh-pages: true +# fail-on-alert: true +# summary-always: true +# github-token: ${{ secrets.GITHUB_TOKEN }} +# comment-always: true +# - run: node ./dist/scripts/ci_validate_modification.js before_data.js 'Benchmark.js Benchmark' +# +# catch2-framework: +# name: Run Catch2 C++ Benchmark Framework example +# runs-on: ubuntu-20.04 +# steps: +# - uses: actions/checkout@v4 +# - uses: actions/setup-node@v4 +# with: +# node-version: 20 +# cache: 'npm' +# - run: npm ci +# - run: npm run build +# - name: Run benchmark +# run: | +# cd examples/catch2 +# mkdir build && cd build +# cmake -DCMAKE_BUILD_TYPE=Release .. +# cmake --build . --config Release +# ./Catch2_bench > ../benchmark_result.txt +# - name: Save previous data.js +# run: | +# git fetch origin gh-pages +# git checkout gh-pages +# cp ./dev/bench/data.js before_data.js +# git checkout - +# - name: Store benchmark result +# uses: ./ +# with: +# name: Catch2 Benchmark +# tool: 'catch2' +# output-file-path: examples/catch2/benchmark_result.txt +# skip-fetch-gh-pages: true +# fail-on-alert: true +# summary-always: true +# github-token: ${{ secrets.GITHUB_TOKEN }} +# comment-always: true +# - run: node ./dist/scripts/ci_validate_modification.js before_data.js 'Catch2 Benchmark' +# +# cpp-framework: +# name: Run Google C++ Benchmark Framework example +# runs-on: ubuntu-20.04 +# steps: +# - uses: actions/checkout@v4 +# - uses: actions/setup-node@v4 +# with: +# node-version: 20 +# cache: 'npm' +# - run: npm ci +# - run: npm run build +# - name: Cache Benchmark library +# uses: actions/cache@v4 +# with: +# path: examples/cpp/benchmark +# key: ${{ runner.os }}-googlebenchmark-v1.5.0 +# - name: Run benchmark +# run: | +# cd examples/cpp +# make json +# - name: Save previous data.js +# run: | +# git fetch origin gh-pages +# git checkout gh-pages +# cp ./dev/bench/data.js before_data.js +# git checkout - +# - name: Store benchmark result +# uses: ./ +# with: +# name: C++ Benchmark +# tool: 'googlecpp' +# output-file-path: examples/cpp/benchmark_result.json +# skip-fetch-gh-pages: true +# fail-on-alert: true +# summary-always: true +# github-token: ${{ secrets.GITHUB_TOKEN }} +# comment-always: true +# - run: node ./dist/scripts/ci_validate_modification.js before_data.js 'C++ Benchmark' +# +# go: +# name: Run Go benchmark example +# runs-on: ubuntu-latest +# steps: +# - uses: actions/checkout@v4 +# - uses: actions/setup-node@v4 +# with: +# node-version: 20 +# cache: 'npm' +# - uses: actions/setup-go@v4 +# with: +# go-version: "stable" +# - run: npm ci +# - run: npm run build +# - name: Run benchmark +# run: cd examples/go && go test -bench 'BenchmarkFib' | tee output.txt +# - name: Save previous data.js +# run: | +# git fetch origin gh-pages +# git checkout gh-pages +# cp ./dev/bench/data.js before_data.js +# git checkout - +# - name: Store benchmark result +# uses: ./ +# with: +# name: Go Benchmark +# tool: 'go' +# output-file-path: examples/go/output.txt +# skip-fetch-gh-pages: true +# fail-on-alert: true +# summary-always: true +# github-token: ${{ secrets.GITHUB_TOKEN }} +# comment-always: true +# - run: node ./dist/scripts/ci_validate_modification.js before_data.js 'Go Benchmark' +# +# java-jmh: +# name: Run JMH Java Benchmark Framework example +# runs-on: ubuntu-latest +# steps: +# - uses: actions/checkout@v4 +# - uses: actions/setup-node@v4 +# with: +# node-version: 20 +# cache: 'npm' +# - uses: actions/setup-java@v4 +# with: +# distribution: 'adopt' +# java-version: '11' +# - run: npm ci +# - run: npm run build +# - name: Run benchmark +# run: | +# cd examples/java +# mvn clean verify +# java -jar target/benchmarks.jar -wi 1 -i 3 -f 1 -rf json +# - name: Save previous data.js +# run: | +# git fetch origin gh-pages +# git checkout gh-pages +# cp ./dev/bench/data.js before_data.js +# git checkout - +# - name: Store benchmark result +# uses: ./ +# with: +# name: JMH Benchmark +# tool: "jmh" +# output-file-path: examples/java/jmh-result.json +# skip-fetch-gh-pages: true +# fail-on-alert: true +# summary-always: true +# github-token: ${{ secrets.GITHUB_TOKEN }} +# comment-always: true +# - run: node ./dist/scripts/ci_validate_modification.js before_data.js 'JMH Benchmark' +# +# julia-benchmark: +# name: Run Julia benchmark example +# runs-on: ubuntu-latest +# steps: +# - uses: actions/checkout@v4 +# - uses: actions/setup-node@v4 +# with: +# node-version: 20 +# cache: 'npm' +# - run: npm ci +# - run: npm run build +# - uses: julia-actions/setup-julia@v1 +# with: +# version: '1' +# arch: x64 +# - name: Run benchmark +# run: | +# cd examples/julia +# julia --project --color=yes -e ' +# using Pkg; +# Pkg.instantiate(); +# include("fib.jl")' +# - name: Save previous data.js +# run: | +# git fetch origin gh-pages +# git checkout gh-pages +# cp ./dev/bench/data.js before_data.js +# git checkout - +# - name: Store benchmark result +# uses: ./ +# with: +# name: Julia benchmark +# tool: 'julia' +# output-file-path: examples/julia/output.json +# skip-fetch-gh-pages: true +# fail-on-alert: true +# summary-always: true +# github-token: ${{ secrets.GITHUB_TOKEN }} +# comment-always: true +# - run: node ./dist/scripts/ci_validate_modification.js before_data.js 'Julia benchmark' +# +# pytest-benchmark: +# name: Run Pytest benchmark example +# runs-on: ubuntu-latest +# steps: +# - uses: actions/checkout@v4 +# - uses: actions/setup-node@v4 +# with: +# node-version: 20 +# cache: 'npm' +# - uses: actions/setup-python@v5 +# with: +# python-version: 3.9 +# - run: npm ci +# - run: npm run build +# - name: Run benchmark +# run: | +# cd examples/pytest +# pip install -r requirements.txt +# pytest bench.py --benchmark-json output.json +# - name: Save previous data.js +# run: | +# git fetch origin gh-pages +# git checkout gh-pages +# cp ./dev/bench/data.js before_data.js +# git checkout - +# - name: Store benchmark result +# uses: ./ +# with: +# name: Python Benchmark with pytest-benchmark +# tool: 'pytest' +# output-file-path: examples/pytest/output.json +# skip-fetch-gh-pages: true +# fail-on-alert: true +# summary-always: true +# github-token: ${{ secrets.GITHUB_TOKEN }} +# comment-always: true +# - run: node ./dist/scripts/ci_validate_modification.js before_data.js 'Python Benchmark with pytest-benchmark' +# +# rust: +# name: Run Rust benchmark example +# runs-on: ubuntu-latest +# steps: +# - uses: actions/checkout@v4 +# - uses: actions/setup-node@v4 +# with: +# node-version: 20 +# cache: 'npm' +# - run: npm ci +# - run: npm run build +# - run: rustup toolchain update nightly && rustup default nightly +# - name: Run benchmark +# run: cd examples/rust && cargo +nightly bench | tee output.txt +# - name: Save previous data.js +# run: | +# git fetch origin gh-pages +# git checkout gh-pages +# cp ./dev/bench/data.js before_data.js +# git checkout - +# - name: Store benchmark result +# uses: ./ +# with: +# name: Rust Benchmark +# tool: 'cargo' +# output-file-path: examples/rust/output.txt +# fail-on-alert: true +# summary-always: true +# github-token: ${{ secrets.GITHUB_TOKEN }} +# comment-always: true +# - run: node ./dist/scripts/ci_validate_modification.js before_data.js 'Rust Benchmark' rust-criterion-rs-framework: name: Run Criterion.rs benchmark example @@ -360,7 +360,7 @@ jobs: cp ./dev/bench/data.js before_data.js git checkout - - name: Store benchmark result - uses: benchmark-action/github-action-benchmark@v1 + uses: ./ with: name: Criterion.rs Benchmark tool: 'cargo' @@ -371,65 +371,65 @@ jobs: comment-always: true - run: node ./dist/scripts/ci_validate_modification.js before_data.js 'Criterion.rs Benchmark' - only-alert-with-cache: - name: Run alert check with actions/cache - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-node@v4 - with: - node-version: 20 - cache: 'npm' - - run: npm ci - - run: npm run build - - uses: actions/setup-go@v4 - with: - go-version: "stable" - - name: Run benchmark - run: cd examples/go && go test -bench 'BenchmarkFib' | tee output.txt - - name: Download previous benchmark data - uses: actions/cache@v4 - with: - path: ./cache - key: ${{ runner.os }}-ci-cache-test - - name: Store benchmark result - uses: ./ - with: - name: Alert setup example with cache - tool: 'go' - output-file-path: examples/go/output.txt - external-data-json-path: ./cache/benchmark-data.json - github-token: ${{ secrets.GITHUB_TOKEN }} - alert-threshold: '200%' - comment-on-alert: true - fail-on-alert: true - alert-comment-cc-users: '@ktrz' - unit-tests: - name: Run unit tests - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-node@v4 - with: - node-version: 20 - cache: 'npm' - - run: npm ci - - run: npm run coverage - - name: Apply yamllint - run: | - sudo pip install yamllint - yamllint --strict .github/workflows - - uses: codecov/codecov-action@v2 - - lint: - name: Run linting and formatting check - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-node@v4 - with: - node-version: 20 - cache: 'npm' - - run: npm ci - - run: npm run lint - - run: npm run format:check +# only-alert-with-cache: +# name: Run alert check with actions/cache +# runs-on: ubuntu-latest +# steps: +# - uses: actions/checkout@v4 +# - uses: actions/setup-node@v4 +# with: +# node-version: 20 +# cache: 'npm' +# - run: npm ci +# - run: npm run build +# - uses: actions/setup-go@v4 +# with: +# go-version: "stable" +# - name: Run benchmark +# run: cd examples/go && go test -bench 'BenchmarkFib' | tee output.txt +# - name: Download previous benchmark data +# uses: actions/cache@v4 +# with: +# path: ./cache +# key: ${{ runner.os }}-ci-cache-test +# - name: Store benchmark result +# uses: ./ +# with: +# name: Alert setup example with cache +# tool: 'go' +# output-file-path: examples/go/output.txt +# external-data-json-path: ./cache/benchmark-data.json +# github-token: ${{ secrets.GITHUB_TOKEN }} +# alert-threshold: '200%' +# comment-on-alert: true +# fail-on-alert: true +# alert-comment-cc-users: '@ktrz' +# unit-tests: +# name: Run unit tests +# runs-on: ubuntu-latest +# steps: +# - uses: actions/checkout@v4 +# - uses: actions/setup-node@v4 +# with: +# node-version: 20 +# cache: 'npm' +# - run: npm ci +# - run: npm run coverage +# - name: Apply yamllint +# run: | +# sudo pip install yamllint +# yamllint --strict .github/workflows +# - uses: codecov/codecov-action@v2 +# +# lint: +# name: Run linting and formatting check +# runs-on: ubuntu-latest +# steps: +# - uses: actions/checkout@v4 +# - uses: actions/setup-node@v4 +# with: +# node-version: 20 +# cache: 'npm' +# - run: npm ci +# - run: npm run lint +# - run: npm run format:check From e5c39e5f5614a3bd07d034a5eff9c942e4277f24 Mon Sep 17 00:00:00 2001 From: Chris Trzesniewski Date: Wed, 27 Mar 2024 10:37:21 +0100 Subject: [PATCH 23/34] cleanup --- .github/workflows/ci-results-repo.yml | 6 +- .github/workflows/ci.yml | 778 +++++++++++++------------- src/git.ts | 2 - src/write.ts | 21 +- 4 files changed, 399 insertions(+), 408 deletions(-) diff --git a/.github/workflows/ci-results-repo.yml b/.github/workflows/ci-results-repo.yml index ba87173f4..5a1ca41de 100644 --- a/.github/workflows/ci-results-repo.yml +++ b/.github/workflows/ci-results-repo.yml @@ -3,9 +3,9 @@ on: push: branches: - master -# pull_request: -# branches: -# - master + pull_request: + branches: + - master jobs: benchmarkdotnet-framework: diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c025d5913..b8509314f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -11,333 +11,333 @@ permissions: pull-requests: write jobs: -# benchmarkdotnet-framework: -# name: Run Benchmark.Net .Net Benchmark Framework example -# runs-on: ubuntu-latest -# steps: -# - uses: actions/checkout@v4 -# - uses: actions/setup-node@v4 -# with: -# node-version: 20 -# cache: 'npm' -# - uses: actions/setup-dotnet@v3 -# with: -# dotnet-version: '6.0.101' # SDK Version to use. keep in line with examples/benchmarkdotnet/global.json -# - run: npm ci -# - run: npm run build -# - name: Run benchmark -# run: cd examples/benchmarkdotnet && dotnet run --exporters json --filter '*' -# - name: Save previous data.js -# run: | -# git fetch origin gh-pages -# git checkout gh-pages -# cp ./dev/bench/data.js before_data.js -# git checkout - -# - name: Store benchmark result -# uses: ./ -# with: -# name: Benchmark.Net Benchmark -# tool: 'benchmarkdotnet' -# output-file-path: examples/benchmarkdotnet/BenchmarkDotNet.Artifacts/results/Sample.Benchmarks-report-full-compressed.json -# skip-fetch-gh-pages: true -# fail-on-alert: true -# summary-always: true -# github-token: ${{ secrets.GITHUB_TOKEN }} -# comment-always: true -# - run: node ./dist/scripts/ci_validate_modification.js before_data.js 'Benchmark.Net Benchmark' -# -# benchmarkjs: -# name: Run JavaScript benchmark example -# runs-on: ubuntu-latest -# steps: -# - uses: actions/checkout@v4 -# - uses: actions/setup-node@v4 -# with: -# node-version: 20 -# cache: 'npm' -# - run: npm ci -# - run: npm run build -# - name: Run benchmark -# run: cd examples/benchmarkjs && npm install && node bench.js | tee output.txt -# - name: Save previous data.js -# run: | -# git fetch origin gh-pages -# git checkout gh-pages -# cp ./dev/bench/data.js before_data.js -# git checkout - -# - name: Store benchmark result -# uses: ./ -# with: -# name: Benchmark.js Benchmark -# tool: 'benchmarkjs' -# output-file-path: examples/benchmarkjs/output.txt -# skip-fetch-gh-pages: true -# fail-on-alert: true -# summary-always: true -# github-token: ${{ secrets.GITHUB_TOKEN }} -# comment-always: true -# - run: node ./dist/scripts/ci_validate_modification.js before_data.js 'Benchmark.js Benchmark' -# -# catch2-framework: -# name: Run Catch2 C++ Benchmark Framework example -# runs-on: ubuntu-20.04 -# steps: -# - uses: actions/checkout@v4 -# - uses: actions/setup-node@v4 -# with: -# node-version: 20 -# cache: 'npm' -# - run: npm ci -# - run: npm run build -# - name: Run benchmark -# run: | -# cd examples/catch2 -# mkdir build && cd build -# cmake -DCMAKE_BUILD_TYPE=Release .. -# cmake --build . --config Release -# ./Catch2_bench > ../benchmark_result.txt -# - name: Save previous data.js -# run: | -# git fetch origin gh-pages -# git checkout gh-pages -# cp ./dev/bench/data.js before_data.js -# git checkout - -# - name: Store benchmark result -# uses: ./ -# with: -# name: Catch2 Benchmark -# tool: 'catch2' -# output-file-path: examples/catch2/benchmark_result.txt -# skip-fetch-gh-pages: true -# fail-on-alert: true -# summary-always: true -# github-token: ${{ secrets.GITHUB_TOKEN }} -# comment-always: true -# - run: node ./dist/scripts/ci_validate_modification.js before_data.js 'Catch2 Benchmark' -# -# cpp-framework: -# name: Run Google C++ Benchmark Framework example -# runs-on: ubuntu-20.04 -# steps: -# - uses: actions/checkout@v4 -# - uses: actions/setup-node@v4 -# with: -# node-version: 20 -# cache: 'npm' -# - run: npm ci -# - run: npm run build -# - name: Cache Benchmark library -# uses: actions/cache@v4 -# with: -# path: examples/cpp/benchmark -# key: ${{ runner.os }}-googlebenchmark-v1.5.0 -# - name: Run benchmark -# run: | -# cd examples/cpp -# make json -# - name: Save previous data.js -# run: | -# git fetch origin gh-pages -# git checkout gh-pages -# cp ./dev/bench/data.js before_data.js -# git checkout - -# - name: Store benchmark result -# uses: ./ -# with: -# name: C++ Benchmark -# tool: 'googlecpp' -# output-file-path: examples/cpp/benchmark_result.json -# skip-fetch-gh-pages: true -# fail-on-alert: true -# summary-always: true -# github-token: ${{ secrets.GITHUB_TOKEN }} -# comment-always: true -# - run: node ./dist/scripts/ci_validate_modification.js before_data.js 'C++ Benchmark' -# -# go: -# name: Run Go benchmark example -# runs-on: ubuntu-latest -# steps: -# - uses: actions/checkout@v4 -# - uses: actions/setup-node@v4 -# with: -# node-version: 20 -# cache: 'npm' -# - uses: actions/setup-go@v4 -# with: -# go-version: "stable" -# - run: npm ci -# - run: npm run build -# - name: Run benchmark -# run: cd examples/go && go test -bench 'BenchmarkFib' | tee output.txt -# - name: Save previous data.js -# run: | -# git fetch origin gh-pages -# git checkout gh-pages -# cp ./dev/bench/data.js before_data.js -# git checkout - -# - name: Store benchmark result -# uses: ./ -# with: -# name: Go Benchmark -# tool: 'go' -# output-file-path: examples/go/output.txt -# skip-fetch-gh-pages: true -# fail-on-alert: true -# summary-always: true -# github-token: ${{ secrets.GITHUB_TOKEN }} -# comment-always: true -# - run: node ./dist/scripts/ci_validate_modification.js before_data.js 'Go Benchmark' -# -# java-jmh: -# name: Run JMH Java Benchmark Framework example -# runs-on: ubuntu-latest -# steps: -# - uses: actions/checkout@v4 -# - uses: actions/setup-node@v4 -# with: -# node-version: 20 -# cache: 'npm' -# - uses: actions/setup-java@v4 -# with: -# distribution: 'adopt' -# java-version: '11' -# - run: npm ci -# - run: npm run build -# - name: Run benchmark -# run: | -# cd examples/java -# mvn clean verify -# java -jar target/benchmarks.jar -wi 1 -i 3 -f 1 -rf json -# - name: Save previous data.js -# run: | -# git fetch origin gh-pages -# git checkout gh-pages -# cp ./dev/bench/data.js before_data.js -# git checkout - -# - name: Store benchmark result -# uses: ./ -# with: -# name: JMH Benchmark -# tool: "jmh" -# output-file-path: examples/java/jmh-result.json -# skip-fetch-gh-pages: true -# fail-on-alert: true -# summary-always: true -# github-token: ${{ secrets.GITHUB_TOKEN }} -# comment-always: true -# - run: node ./dist/scripts/ci_validate_modification.js before_data.js 'JMH Benchmark' -# -# julia-benchmark: -# name: Run Julia benchmark example -# runs-on: ubuntu-latest -# steps: -# - uses: actions/checkout@v4 -# - uses: actions/setup-node@v4 -# with: -# node-version: 20 -# cache: 'npm' -# - run: npm ci -# - run: npm run build -# - uses: julia-actions/setup-julia@v1 -# with: -# version: '1' -# arch: x64 -# - name: Run benchmark -# run: | -# cd examples/julia -# julia --project --color=yes -e ' -# using Pkg; -# Pkg.instantiate(); -# include("fib.jl")' -# - name: Save previous data.js -# run: | -# git fetch origin gh-pages -# git checkout gh-pages -# cp ./dev/bench/data.js before_data.js -# git checkout - -# - name: Store benchmark result -# uses: ./ -# with: -# name: Julia benchmark -# tool: 'julia' -# output-file-path: examples/julia/output.json -# skip-fetch-gh-pages: true -# fail-on-alert: true -# summary-always: true -# github-token: ${{ secrets.GITHUB_TOKEN }} -# comment-always: true -# - run: node ./dist/scripts/ci_validate_modification.js before_data.js 'Julia benchmark' -# -# pytest-benchmark: -# name: Run Pytest benchmark example -# runs-on: ubuntu-latest -# steps: -# - uses: actions/checkout@v4 -# - uses: actions/setup-node@v4 -# with: -# node-version: 20 -# cache: 'npm' -# - uses: actions/setup-python@v5 -# with: -# python-version: 3.9 -# - run: npm ci -# - run: npm run build -# - name: Run benchmark -# run: | -# cd examples/pytest -# pip install -r requirements.txt -# pytest bench.py --benchmark-json output.json -# - name: Save previous data.js -# run: | -# git fetch origin gh-pages -# git checkout gh-pages -# cp ./dev/bench/data.js before_data.js -# git checkout - -# - name: Store benchmark result -# uses: ./ -# with: -# name: Python Benchmark with pytest-benchmark -# tool: 'pytest' -# output-file-path: examples/pytest/output.json -# skip-fetch-gh-pages: true -# fail-on-alert: true -# summary-always: true -# github-token: ${{ secrets.GITHUB_TOKEN }} -# comment-always: true -# - run: node ./dist/scripts/ci_validate_modification.js before_data.js 'Python Benchmark with pytest-benchmark' -# -# rust: -# name: Run Rust benchmark example -# runs-on: ubuntu-latest -# steps: -# - uses: actions/checkout@v4 -# - uses: actions/setup-node@v4 -# with: -# node-version: 20 -# cache: 'npm' -# - run: npm ci -# - run: npm run build -# - run: rustup toolchain update nightly && rustup default nightly -# - name: Run benchmark -# run: cd examples/rust && cargo +nightly bench | tee output.txt -# - name: Save previous data.js -# run: | -# git fetch origin gh-pages -# git checkout gh-pages -# cp ./dev/bench/data.js before_data.js -# git checkout - -# - name: Store benchmark result -# uses: ./ -# with: -# name: Rust Benchmark -# tool: 'cargo' -# output-file-path: examples/rust/output.txt -# fail-on-alert: true -# summary-always: true -# github-token: ${{ secrets.GITHUB_TOKEN }} -# comment-always: true -# - run: node ./dist/scripts/ci_validate_modification.js before_data.js 'Rust Benchmark' + benchmarkdotnet-framework: + name: Run Benchmark.Net .Net Benchmark Framework example + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version: 20 + cache: 'npm' + - uses: actions/setup-dotnet@v3 + with: + dotnet-version: '6.0.101' # SDK Version to use. keep in line with examples/benchmarkdotnet/global.json + - run: npm ci + - run: npm run build + - name: Run benchmark + run: cd examples/benchmarkdotnet && dotnet run --exporters json --filter '*' + - name: Save previous data.js + run: | + git fetch origin gh-pages + git checkout gh-pages + cp ./dev/bench/data.js before_data.js + git checkout - + - name: Store benchmark result + uses: ./ + with: + name: Benchmark.Net Benchmark + tool: 'benchmarkdotnet' + output-file-path: examples/benchmarkdotnet/BenchmarkDotNet.Artifacts/results/Sample.Benchmarks-report-full-compressed.json + skip-fetch-gh-pages: true + fail-on-alert: true + summary-always: true + github-token: ${{ secrets.GITHUB_TOKEN }} + comment-always: true + - run: node ./dist/scripts/ci_validate_modification.js before_data.js 'Benchmark.Net Benchmark' + + benchmarkjs: + name: Run JavaScript benchmark example + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version: 20 + cache: 'npm' + - run: npm ci + - run: npm run build + - name: Run benchmark + run: cd examples/benchmarkjs && npm install && node bench.js | tee output.txt + - name: Save previous data.js + run: | + git fetch origin gh-pages + git checkout gh-pages + cp ./dev/bench/data.js before_data.js + git checkout - + - name: Store benchmark result + uses: ./ + with: + name: Benchmark.js Benchmark + tool: 'benchmarkjs' + output-file-path: examples/benchmarkjs/output.txt + skip-fetch-gh-pages: true + fail-on-alert: true + summary-always: true + github-token: ${{ secrets.GITHUB_TOKEN }} + comment-always: true + - run: node ./dist/scripts/ci_validate_modification.js before_data.js 'Benchmark.js Benchmark' + + catch2-framework: + name: Run Catch2 C++ Benchmark Framework example + runs-on: ubuntu-20.04 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version: 20 + cache: 'npm' + - run: npm ci + - run: npm run build + - name: Run benchmark + run: | + cd examples/catch2 + mkdir build && cd build + cmake -DCMAKE_BUILD_TYPE=Release .. + cmake --build . --config Release + ./Catch2_bench > ../benchmark_result.txt + - name: Save previous data.js + run: | + git fetch origin gh-pages + git checkout gh-pages + cp ./dev/bench/data.js before_data.js + git checkout - + - name: Store benchmark result + uses: ./ + with: + name: Catch2 Benchmark + tool: 'catch2' + output-file-path: examples/catch2/benchmark_result.txt + skip-fetch-gh-pages: true + fail-on-alert: true + summary-always: true + github-token: ${{ secrets.GITHUB_TOKEN }} + comment-always: true + - run: node ./dist/scripts/ci_validate_modification.js before_data.js 'Catch2 Benchmark' + + cpp-framework: + name: Run Google C++ Benchmark Framework example + runs-on: ubuntu-20.04 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version: 20 + cache: 'npm' + - run: npm ci + - run: npm run build + - name: Cache Benchmark library + uses: actions/cache@v4 + with: + path: examples/cpp/benchmark + key: ${{ runner.os }}-googlebenchmark-v1.5.0 + - name: Run benchmark + run: | + cd examples/cpp + make json + - name: Save previous data.js + run: | + git fetch origin gh-pages + git checkout gh-pages + cp ./dev/bench/data.js before_data.js + git checkout - + - name: Store benchmark result + uses: ./ + with: + name: C++ Benchmark + tool: 'googlecpp' + output-file-path: examples/cpp/benchmark_result.json + skip-fetch-gh-pages: true + fail-on-alert: true + summary-always: true + github-token: ${{ secrets.GITHUB_TOKEN }} + comment-always: true + - run: node ./dist/scripts/ci_validate_modification.js before_data.js 'C++ Benchmark' + + go: + name: Run Go benchmark example + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version: 20 + cache: 'npm' + - uses: actions/setup-go@v4 + with: + go-version: "stable" + - run: npm ci + - run: npm run build + - name: Run benchmark + run: cd examples/go && go test -bench 'BenchmarkFib' | tee output.txt + - name: Save previous data.js + run: | + git fetch origin gh-pages + git checkout gh-pages + cp ./dev/bench/data.js before_data.js + git checkout - + - name: Store benchmark result + uses: ./ + with: + name: Go Benchmark + tool: 'go' + output-file-path: examples/go/output.txt + skip-fetch-gh-pages: true + fail-on-alert: true + summary-always: true + github-token: ${{ secrets.GITHUB_TOKEN }} + comment-always: true + - run: node ./dist/scripts/ci_validate_modification.js before_data.js 'Go Benchmark' + + java-jmh: + name: Run JMH Java Benchmark Framework example + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version: 20 + cache: 'npm' + - uses: actions/setup-java@v4 + with: + distribution: 'adopt' + java-version: '11' + - run: npm ci + - run: npm run build + - name: Run benchmark + run: | + cd examples/java + mvn clean verify + java -jar target/benchmarks.jar -wi 1 -i 3 -f 1 -rf json + - name: Save previous data.js + run: | + git fetch origin gh-pages + git checkout gh-pages + cp ./dev/bench/data.js before_data.js + git checkout - + - name: Store benchmark result + uses: ./ + with: + name: JMH Benchmark + tool: "jmh" + output-file-path: examples/java/jmh-result.json + skip-fetch-gh-pages: true + fail-on-alert: true + summary-always: true + github-token: ${{ secrets.GITHUB_TOKEN }} + comment-always: true + - run: node ./dist/scripts/ci_validate_modification.js before_data.js 'JMH Benchmark' + + julia-benchmark: + name: Run Julia benchmark example + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version: 20 + cache: 'npm' + - run: npm ci + - run: npm run build + - uses: julia-actions/setup-julia@v1 + with: + version: '1' + arch: x64 + - name: Run benchmark + run: | + cd examples/julia + julia --project --color=yes -e ' + using Pkg; + Pkg.instantiate(); + include("fib.jl")' + - name: Save previous data.js + run: | + git fetch origin gh-pages + git checkout gh-pages + cp ./dev/bench/data.js before_data.js + git checkout - + - name: Store benchmark result + uses: ./ + with: + name: Julia benchmark + tool: 'julia' + output-file-path: examples/julia/output.json + skip-fetch-gh-pages: true + fail-on-alert: true + summary-always: true + github-token: ${{ secrets.GITHUB_TOKEN }} + comment-always: true + - run: node ./dist/scripts/ci_validate_modification.js before_data.js 'Julia benchmark' + + pytest-benchmark: + name: Run Pytest benchmark example + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version: 20 + cache: 'npm' + - uses: actions/setup-python@v5 + with: + python-version: 3.9 + - run: npm ci + - run: npm run build + - name: Run benchmark + run: | + cd examples/pytest + pip install -r requirements.txt + pytest bench.py --benchmark-json output.json + - name: Save previous data.js + run: | + git fetch origin gh-pages + git checkout gh-pages + cp ./dev/bench/data.js before_data.js + git checkout - + - name: Store benchmark result + uses: ./ + with: + name: Python Benchmark with pytest-benchmark + tool: 'pytest' + output-file-path: examples/pytest/output.json + skip-fetch-gh-pages: true + fail-on-alert: true + summary-always: true + github-token: ${{ secrets.GITHUB_TOKEN }} + comment-always: true + - run: node ./dist/scripts/ci_validate_modification.js before_data.js 'Python Benchmark with pytest-benchmark' + + rust: + name: Run Rust benchmark example + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version: 20 + cache: 'npm' + - run: npm ci + - run: npm run build + - run: rustup toolchain update nightly && rustup default nightly + - name: Run benchmark + run: cd examples/rust && cargo +nightly bench | tee output.txt + - name: Save previous data.js + run: | + git fetch origin gh-pages + git checkout gh-pages + cp ./dev/bench/data.js before_data.js + git checkout - + - name: Store benchmark result + uses: ./ + with: + name: Rust Benchmark + tool: 'cargo' + output-file-path: examples/rust/output.txt + fail-on-alert: true + summary-always: true + github-token: ${{ secrets.GITHUB_TOKEN }} + comment-always: true + - run: node ./dist/scripts/ci_validate_modification.js before_data.js 'Rust Benchmark' rust-criterion-rs-framework: name: Run Criterion.rs benchmark example @@ -371,65 +371,65 @@ jobs: comment-always: true - run: node ./dist/scripts/ci_validate_modification.js before_data.js 'Criterion.rs Benchmark' -# only-alert-with-cache: -# name: Run alert check with actions/cache -# runs-on: ubuntu-latest -# steps: -# - uses: actions/checkout@v4 -# - uses: actions/setup-node@v4 -# with: -# node-version: 20 -# cache: 'npm' -# - run: npm ci -# - run: npm run build -# - uses: actions/setup-go@v4 -# with: -# go-version: "stable" -# - name: Run benchmark -# run: cd examples/go && go test -bench 'BenchmarkFib' | tee output.txt -# - name: Download previous benchmark data -# uses: actions/cache@v4 -# with: -# path: ./cache -# key: ${{ runner.os }}-ci-cache-test -# - name: Store benchmark result -# uses: ./ -# with: -# name: Alert setup example with cache -# tool: 'go' -# output-file-path: examples/go/output.txt -# external-data-json-path: ./cache/benchmark-data.json -# github-token: ${{ secrets.GITHUB_TOKEN }} -# alert-threshold: '200%' -# comment-on-alert: true -# fail-on-alert: true -# alert-comment-cc-users: '@ktrz' -# unit-tests: -# name: Run unit tests -# runs-on: ubuntu-latest -# steps: -# - uses: actions/checkout@v4 -# - uses: actions/setup-node@v4 -# with: -# node-version: 20 -# cache: 'npm' -# - run: npm ci -# - run: npm run coverage -# - name: Apply yamllint -# run: | -# sudo pip install yamllint -# yamllint --strict .github/workflows -# - uses: codecov/codecov-action@v2 -# -# lint: -# name: Run linting and formatting check -# runs-on: ubuntu-latest -# steps: -# - uses: actions/checkout@v4 -# - uses: actions/setup-node@v4 -# with: -# node-version: 20 -# cache: 'npm' -# - run: npm ci -# - run: npm run lint -# - run: npm run format:check + only-alert-with-cache: + name: Run alert check with actions/cache + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version: 20 + cache: 'npm' + - run: npm ci + - run: npm run build + - uses: actions/setup-go@v4 + with: + go-version: "stable" + - name: Run benchmark + run: cd examples/go && go test -bench 'BenchmarkFib' | tee output.txt + - name: Download previous benchmark data + uses: actions/cache@v4 + with: + path: ./cache + key: ${{ runner.os }}-ci-cache-test + - name: Store benchmark result + uses: ./ + with: + name: Alert setup example with cache + tool: 'go' + output-file-path: examples/go/output.txt + external-data-json-path: ./cache/benchmark-data.json + github-token: ${{ secrets.GITHUB_TOKEN }} + alert-threshold: '200%' + comment-on-alert: true + fail-on-alert: true + alert-comment-cc-users: '@ktrz' + unit-tests: + name: Run unit tests + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version: 20 + cache: 'npm' + - run: npm ci + - run: npm run coverage + - name: Apply yamllint + run: | + sudo pip install yamllint + yamllint --strict .github/workflows + - uses: codecov/codecov-action@v2 + + lint: + name: Run linting and formatting check + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version: 20 + cache: 'npm' + - run: npm ci + - run: npm run lint + - run: npm run format:check diff --git a/src/git.ts b/src/git.ts index df47a3add..1a228ac6d 100644 --- a/src/git.ts +++ b/src/git.ts @@ -39,7 +39,6 @@ async function capture(cmd: string, args: string[]): Promise { } export function getServerUrlObj(repositoryUrl: string | undefined): URL { - core.warning(`getServerUrlObj ${repositoryUrl}`); const urlValue = repositoryUrl && repositoryUrl.trim().length > 0 ? repositoryUrl @@ -48,7 +47,6 @@ export function getServerUrlObj(repositoryUrl: string | undefined): URL { } export function getServerUrl(repositoryUrl: string | undefined): string { - core.warning(`getServerUrl ${repositoryUrl}`); return getServerUrlObj(repositoryUrl).origin; } diff --git a/src/write.ts b/src/write.ts index b41c262a4..133d0bc58 100644 --- a/src/write.ts +++ b/src/write.ts @@ -120,7 +120,6 @@ function findAlerts(curSuite: Benchmark, prevSuite: Benchmark, threshold: number } function getCurrentRepoMetadata() { - core.warning('getCurrentRepoMetadata'); const { repo, owner } = github.context.repo; const serverUrl = git.getServerUrl(github.context.payload.repository?.html_url); return { @@ -240,20 +239,14 @@ function buildAlertComment( } async function leaveComment(commitId: string, body: string, commentId: string, token: string) { - try { - core.warning('Sending comment:\n' + body); - core.warning('Boo'); + core.debug('Sending comment:\n' + body); - const repoMetadata = getCurrentRepoMetadata(); - const pr = github.context.payload.pull_request; + const repoMetadata = getCurrentRepoMetadata(); + const pr = github.context.payload.pull_request; - return await (pr?.number - ? leavePRComment(repoMetadata.owner.login, repoMetadata.name, pr.number, body, commentId, token) - : leaveCommitComment(repoMetadata.owner.login, repoMetadata.name, commitId, body, commentId, token)); - } catch (e) { - core.warning(`error: ${e}`); - return; - } + return await (pr?.number + ? leavePRComment(repoMetadata.owner.login, repoMetadata.name, pr.number, body, commentId, token) + : leaveCommitComment(repoMetadata.owner.login, repoMetadata.name, commitId, body, commentId, token)); } async function handleComment(benchName: string, curSuite: Benchmark, prevSuite: Benchmark, config: Config) { @@ -299,7 +292,7 @@ async function handleAlert(benchName: string, curSuite: Benchmark, prevSuite: Be throw new Error("'comment-on-alert' input is set but 'github-token' input is not set"); } const res = await leaveComment(curSuite.commit.id, body, `${benchName} Alert`, githubToken); - url = res?.data.html_url; + url = res.data.html_url; message = body + `\nComment was generated at ${url}`; } From dc8231a1a26e21bcc5b1b64b34b591435adc8252 Mon Sep 17 00:00:00 2001 From: Chris Trzesniewski Date: Wed, 27 Mar 2024 10:45:14 +0100 Subject: [PATCH 24/34] check if permissions needed --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b8509314f..85c0529e2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -7,8 +7,8 @@ on: branches: - master -permissions: - pull-requests: write +#permissions: +# pull-requests: write jobs: benchmarkdotnet-framework: From 49be1e5b506737d9cf1c391b74c3511e4409ba8c Mon Sep 17 00:00:00 2001 From: Chris Trzesniewski Date: Wed, 27 Mar 2024 10:48:16 +0100 Subject: [PATCH 25/34] cleanup --- .github/workflows/ci.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 85c0529e2..539eaedb5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -7,9 +7,6 @@ on: branches: - master -#permissions: -# pull-requests: write - jobs: benchmarkdotnet-framework: name: Run Benchmark.Net .Net Benchmark Framework example From 378418e2c5ee625cb5eec4cbe88043425ddaf4e6 Mon Sep 17 00:00:00 2001 From: Chris Trzesniewski Date: Wed, 27 Mar 2024 10:55:15 +0100 Subject: [PATCH 26/34] comment on alert --- .github/workflows/ci.yml | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 539eaedb5..071287f70 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -40,7 +40,7 @@ jobs: fail-on-alert: true summary-always: true github-token: ${{ secrets.GITHUB_TOKEN }} - comment-always: true + comment-on-alert: true - run: node ./dist/scripts/ci_validate_modification.js before_data.js 'Benchmark.Net Benchmark' benchmarkjs: @@ -72,7 +72,7 @@ jobs: fail-on-alert: true summary-always: true github-token: ${{ secrets.GITHUB_TOKEN }} - comment-always: true + comment-on-alert: true - run: node ./dist/scripts/ci_validate_modification.js before_data.js 'Benchmark.js Benchmark' catch2-framework: @@ -109,7 +109,7 @@ jobs: fail-on-alert: true summary-always: true github-token: ${{ secrets.GITHUB_TOKEN }} - comment-always: true + comment-on-alert: true - run: node ./dist/scripts/ci_validate_modification.js before_data.js 'Catch2 Benchmark' cpp-framework: @@ -148,7 +148,7 @@ jobs: fail-on-alert: true summary-always: true github-token: ${{ secrets.GITHUB_TOKEN }} - comment-always: true + comment-on-alert: true - run: node ./dist/scripts/ci_validate_modification.js before_data.js 'C++ Benchmark' go: @@ -183,7 +183,7 @@ jobs: fail-on-alert: true summary-always: true github-token: ${{ secrets.GITHUB_TOKEN }} - comment-always: true + comment-on-alert: true - run: node ./dist/scripts/ci_validate_modification.js before_data.js 'Go Benchmark' java-jmh: @@ -222,7 +222,7 @@ jobs: fail-on-alert: true summary-always: true github-token: ${{ secrets.GITHUB_TOKEN }} - comment-always: true + comment-on-alert: true - run: node ./dist/scripts/ci_validate_modification.js before_data.js 'JMH Benchmark' julia-benchmark: @@ -263,7 +263,7 @@ jobs: fail-on-alert: true summary-always: true github-token: ${{ secrets.GITHUB_TOKEN }} - comment-always: true + comment-on-alert: true - run: node ./dist/scripts/ci_validate_modification.js before_data.js 'Julia benchmark' pytest-benchmark: @@ -301,7 +301,7 @@ jobs: fail-on-alert: true summary-always: true github-token: ${{ secrets.GITHUB_TOKEN }} - comment-always: true + comment-on-alert: true - run: node ./dist/scripts/ci_validate_modification.js before_data.js 'Python Benchmark with pytest-benchmark' rust: @@ -333,7 +333,7 @@ jobs: fail-on-alert: true summary-always: true github-token: ${{ secrets.GITHUB_TOKEN }} - comment-always: true + comment-on-alert: true - run: node ./dist/scripts/ci_validate_modification.js before_data.js 'Rust Benchmark' rust-criterion-rs-framework: @@ -365,7 +365,7 @@ jobs: fail-on-alert: true summary-always: true github-token: ${{ secrets.GITHUB_TOKEN }} - comment-always: true + comment-on-alert: true - run: node ./dist/scripts/ci_validate_modification.js before_data.js 'Criterion.rs Benchmark' only-alert-with-cache: From e1ab9068d7fb3815d708e79d5c73871a98835724 Mon Sep 17 00:00:00 2001 From: Chris Trzesniewski Date: Wed, 27 Mar 2024 10:55:24 +0100 Subject: [PATCH 27/34] break bench to test --- examples/benchmarkjs/bench.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/benchmarkjs/bench.js b/examples/benchmarkjs/bench.js index 2cc518338..79a1d5cd8 100644 --- a/examples/benchmarkjs/bench.js +++ b/examples/benchmarkjs/bench.js @@ -4,7 +4,7 @@ const { fib } = require('./index'); suite .add('fib(10)', () => { - fib(10); + fib(15); }) .add('fib(20)', () => { fib(20); From c2148aad0df827a95dbab80095868e439c3ab2de Mon Sep 17 00:00:00 2001 From: Chris Trzesniewski Date: Wed, 27 Mar 2024 10:56:27 +0100 Subject: [PATCH 28/34] break bench to test --- examples/benchmarkjs/bench.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/benchmarkjs/bench.js b/examples/benchmarkjs/bench.js index 79a1d5cd8..0cde87eed 100644 --- a/examples/benchmarkjs/bench.js +++ b/examples/benchmarkjs/bench.js @@ -4,7 +4,7 @@ const { fib } = require('./index'); suite .add('fib(10)', () => { - fib(15); + fib(16); }) .add('fib(20)', () => { fib(20); From 213cd6ae5ae65a87bfb1f47c7bdbda3f177a108e Mon Sep 17 00:00:00 2001 From: Chris Trzesniewski Date: Wed, 27 Mar 2024 10:58:01 +0100 Subject: [PATCH 29/34] fix bench --- examples/benchmarkjs/bench.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/benchmarkjs/bench.js b/examples/benchmarkjs/bench.js index 0cde87eed..2cc518338 100644 --- a/examples/benchmarkjs/bench.js +++ b/examples/benchmarkjs/bench.js @@ -4,7 +4,7 @@ const { fib } = require('./index'); suite .add('fib(10)', () => { - fib(16); + fib(10); }) .add('fib(20)', () => { fib(20); From 0e531883ba9a030b88e7d5ec8c77a291f4994c1d Mon Sep 17 00:00:00 2001 From: Chris Trzesniewski Date: Wed, 27 Mar 2024 10:58:57 +0100 Subject: [PATCH 30/34] break other bench --- examples/benchmarkdotnet/Benchmarks.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/benchmarkdotnet/Benchmarks.cs b/examples/benchmarkdotnet/Benchmarks.cs index d9f79c816..7d7e261be 100644 --- a/examples/benchmarkdotnet/Benchmarks.cs +++ b/examples/benchmarkdotnet/Benchmarks.cs @@ -15,7 +15,7 @@ public static int Fib(int n) { } [Benchmark] - public void Fib10() => Fib(10); + public void Fib10() => Fib(15); [Benchmark] public void Fib20() => Fib(20); From b7f1fa1632059d28fcb5e10db14e6f319973d7e0 Mon Sep 17 00:00:00 2001 From: Chris Trzesniewski Date: Wed, 27 Mar 2024 11:01:54 +0100 Subject: [PATCH 31/34] break other bench --- examples/benchmarkdotnet/Benchmarks.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/benchmarkdotnet/Benchmarks.cs b/examples/benchmarkdotnet/Benchmarks.cs index 7d7e261be..e608d1780 100644 --- a/examples/benchmarkdotnet/Benchmarks.cs +++ b/examples/benchmarkdotnet/Benchmarks.cs @@ -15,7 +15,7 @@ public static int Fib(int n) { } [Benchmark] - public void Fib10() => Fib(15); + public void Fib10() => Fib(20); [Benchmark] public void Fib20() => Fib(20); From 362fe82d74e6c9d74bcc49c776e90910234c2e56 Mon Sep 17 00:00:00 2001 From: Chris Trzesniewski Date: Wed, 27 Mar 2024 11:04:03 +0100 Subject: [PATCH 32/34] break even more --- examples/benchmarkdotnet/Benchmarks.cs | 2 +- examples/benchmarkjs/bench.js | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/benchmarkdotnet/Benchmarks.cs b/examples/benchmarkdotnet/Benchmarks.cs index e608d1780..968d8a993 100644 --- a/examples/benchmarkdotnet/Benchmarks.cs +++ b/examples/benchmarkdotnet/Benchmarks.cs @@ -15,7 +15,7 @@ public static int Fib(int n) { } [Benchmark] - public void Fib10() => Fib(20); + public void Fib10() => Fib(21); [Benchmark] public void Fib20() => Fib(20); diff --git a/examples/benchmarkjs/bench.js b/examples/benchmarkjs/bench.js index 2cc518338..8164e29b1 100644 --- a/examples/benchmarkjs/bench.js +++ b/examples/benchmarkjs/bench.js @@ -4,7 +4,7 @@ const { fib } = require('./index'); suite .add('fib(10)', () => { - fib(10); + fib(20); }) .add('fib(20)', () => { fib(20); From a87e362f37a07eb6d1ee1cd31c42557345ef4569 Mon Sep 17 00:00:00 2001 From: Chris Trzesniewski Date: Wed, 27 Mar 2024 11:06:04 +0100 Subject: [PATCH 33/34] fix benches --- examples/benchmarkdotnet/Benchmarks.cs | 2 +- examples/benchmarkjs/bench.js | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/benchmarkdotnet/Benchmarks.cs b/examples/benchmarkdotnet/Benchmarks.cs index 968d8a993..d9f79c816 100644 --- a/examples/benchmarkdotnet/Benchmarks.cs +++ b/examples/benchmarkdotnet/Benchmarks.cs @@ -15,7 +15,7 @@ public static int Fib(int n) { } [Benchmark] - public void Fib10() => Fib(21); + public void Fib10() => Fib(10); [Benchmark] public void Fib20() => Fib(20); diff --git a/examples/benchmarkjs/bench.js b/examples/benchmarkjs/bench.js index 8164e29b1..2cc518338 100644 --- a/examples/benchmarkjs/bench.js +++ b/examples/benchmarkjs/bench.js @@ -4,7 +4,7 @@ const { fib } = require('./index'); suite .add('fib(10)', () => { - fib(20); + fib(10); }) .add('fib(20)', () => { fib(20); From fddd1b8a5c640eeef5c4b3e95cfb7dd6d465da10 Mon Sep 17 00:00:00 2001 From: Chris Trzesniewski Date: Thu, 28 Mar 2024 08:32:17 +0100 Subject: [PATCH 34/34] add README --- CHANGELOG.md | 1 + src/write.ts | 7 +++---- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b6b3225c5..f503963f1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,6 @@ ## Unreleased - **fix** Rust benchmarks not comparing to baseline (#235) +- **feat** Comment on PR and auto update comment (#223) # [v1.19.3](https://github.com/benchmark-action/github-action-benchmark/releases/tag/v1.19.3) - 02 Feb 2024 diff --git a/src/write.ts b/src/write.ts index 133d0bc58..27e13798e 100644 --- a/src/write.ts +++ b/src/write.ts @@ -285,14 +285,13 @@ async function handleAlert(benchName: string, curSuite: Benchmark, prevSuite: Be core.debug(`Found ${alerts.length} alerts`); const body = buildAlertComment(alerts, benchName, curSuite, prevSuite, alertThreshold, alertCommentCcUsers); let message = body; - let url = null; if (commentOnAlert) { if (!githubToken) { throw new Error("'comment-on-alert' input is set but 'github-token' input is not set"); } const res = await leaveComment(curSuite.commit.id, body, `${benchName} Alert`, githubToken); - url = res.data.html_url; + const url = res.data.html_url; message = body + `\nComment was generated at ${url}`; } @@ -357,7 +356,7 @@ function addBenchmarkToDataJson( return prevBench; } -function isRemoteRejectedError(err: unknown) { +function isRemoteRejectedError(err: unknown): err is Error { if (err instanceof Error) { return ['[remote rejected]', '[rejected]'].some((l) => err.message.includes(l)); } @@ -436,7 +435,7 @@ async function writeBenchmarkToGitHubPagesWithRetry( console.log( `Automatically pushed the generated commit to ${ghPagesBranch} branch since 'auto-push' is set to true`, ); - } catch (err: any) { + } catch (err: unknown) { if (!isRemoteRejectedError(err)) { throw err; }