diff --git a/.github/workflows/cabal.yml b/.github/workflows/cabal.yml new file mode 100644 index 000000000..84a0f7368 --- /dev/null +++ b/.github/workflows/cabal.yml @@ -0,0 +1,48 @@ +name: Cabal Example +on: + push: + branches: + - master + +permissions: + contents: write + deployments: write + +jobs: + benchmark: + name: Run Cabal benchmark example + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - uses: haskell/actions/setup@v2.0 + - name: Run benchmark + run: cd examples/cabal && cabal run -- --csv output.txt + + - name: Store benchmark result + uses: benchmark-action/github-action-benchmark@v1 + with: + name: Cabal Benchmark + tool: 'cabal' + output-file-path: examples/cabal/output.txt + github-token: ${{ secrets.GITHUB_TOKEN }} + auto-push: true + # Show alert with commit comment on detecting possible performance regression + alert-threshold: '200%' + comment-on-alert: true + fail-on-alert: true + alert-comment-cc-users: '@charrsky' + + - name: Store benchmark result - separate results repo + uses: benchmark-action/github-action-benchmark@v1 + with: + name: Cabal Benchmark + tool: 'cabal' + output-file-path: examples/cabal/output.txt + github-token: ${{ secrets.BENCHMARK_ACTION_BOT_TOKEN }} + auto-push: true + # Show alert with commit comment on detecting possible performance regression + alert-threshold: '200%' + comment-on-alert: true + fail-on-alert: true + alert-comment-cc-users: '@charrsky' + gh-repository: 'github.com/benchmark-action/github-action-benchmark-results' \ No newline at end of file diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d04c3021d..2be2ee21d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -293,6 +293,39 @@ jobs: fail-on-alert: true - run: node ./dist/scripts/ci_validate_modification.js before_data.js 'Benchmark.Net Benchmark' + cabal: + name: Run Cabal benchmark example + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-node@v3 + with: + node-version: 16 + - uses: haskell/actions/setup@v2 + - uses: actions/cache@v1 + with: + path: ~/.npm + key: ${{ runner.os }}-node-${{ hashFiles('package-lock.json') }} + - run: npm ci + - run: npm run build + - name: Save previous data.js + run: | + git fetch origin gh-pages + git checkout gh-pages + cp ./dev/bench/data.js before_data.js + git checkout - + - name: Run benchmark + run: cd examples/cabal && cabal run -- --csv output.txt + - name: Store benchmark result + uses: ./ + with: + name: Cabal Benchmark + tool: 'cabal' + output-file-path: examples/cabal/output.txt + skip-fetch-gh-pages: true + fail-on-alert: true + - run: node ./dist/scripts/ci_validate_modification.js before_data.js 'Go Benchmark' + only-alert-with-cache: name: Run alert check with actions/cache runs-on: ubuntu-latest diff --git a/README.md b/README.md index 2e6d8437d..f45906fde 100644 --- a/README.md +++ b/README.md @@ -25,6 +25,7 @@ This action currently supports the following tools: - [BenchmarkTools.jl][] for Julia packages - [Benchmark.Net][benchmarkdotnet] for .Net projects - [benchmarkluau](https://github.com/Roblox/luau/tree/master/bench) for Luau projects +- [criterion](https://hackage.haskell.org/package/criterion) for Haskell projects - Custom benchmarks where either 'biggerIsBetter' or 'smallerIsBetter' Multiple languages in the same repository are supported for polyglot projects. @@ -46,9 +47,11 @@ definitions are in [.github/workflows/](./.github/workflows) directory. Live wor | Python | [![pytest-benchmark Example Workflow][pytest-benchmark-badge]][pytest-workflow-example] | [examples/pytest](./examples/pytest) | | C++ | [![C++ Example Workflow][cpp-badge]][cpp-workflow-example] | [examples/cpp](./examples/cpp) | | C++ (Catch2) | [![C++ Catch2 Example Workflow][catch2-badge]][catch2-workflow-example] | [examples/catch2](./examples/catch2) | -| Julia | [![Julia Example][julia-badge]][julia-workflow-example] | [examples/julia](./examples/julia) | +| Julia | [![Julia Example][julia-badge]][julia-workflow-example] | [examples/julia](./examples/julia) | | .Net | [![C# Benchmark.Net Example Workflow][benchmarkdotnet-badge]][benchmarkdotnet-workflow-example] | [examples/benchmarkdotnet](./examples/benchmarkdotnet) | | Luau | Coming soon | Coming soon | +| Haskell | | | + All benchmark charts from above workflows are gathered in GitHub pages: diff --git a/examples/cabal/Fibber.hs b/examples/cabal/Fibber.hs new file mode 100644 index 000000000..68897683a --- /dev/null +++ b/examples/cabal/Fibber.hs @@ -0,0 +1,20 @@ +-- The simplest/silliest of all benchmarks! + +import Criterion.Main + +fib :: Integer -> Integer +fib m | m < 0 = error "negative!" + | otherwise = go m + where + go 0 = 0 + go 1 = 1 + go n = go (n-1) + go (n-2) + +main :: IO () +main = defaultMain [ + bgroup "fib" [ bench "1" $ whnf fib 1 + , bench "5" $ whnf fib 5 + , bench "9" $ whnf fib 9 + , bench "11" $ whnf fib 11 + ] + ] diff --git a/examples/cabal/README.md b/examples/cabal/README.md new file mode 100644 index 000000000..ad88fd09e --- /dev/null +++ b/examples/cabal/README.md @@ -0,0 +1,43 @@ +Cabal example for benchmarking with `criterion` +================================================= + +- [Workflow for this example](../../.github/workflows/cabal.yml) +- [Action log of this example](https://github.com/benchmark-action/github-action-benchmark/) +- [Benchmark results on GitHub pages](https://benchmark-action.github.io/github-action-benchmark/dev/bench/) + +This directory shows how to use [`github-action-benchmark`](https://github.com/benchmark-action/github-action-benchmark) +with `criterion` package. + +You can see a quick `criterion` tutorial [here](http://www.serpentine.com/criterion/tutorial.html). + +NB: As for now, this GitHub action only supports benchmarks made with `criterion` package outputted as a .csv file. + +## Run benchmarks + +If the file with the benchmarks is also a "benchmark" according to your .cabal file, then + +```yaml +- name: Run benchmark + run: cabal bench --benchmark-options="--csv " +``` + +should do the trick. If it is an "executable" (as it is in the example), then run it with + +```yaml +- name: Run benchmark + run: cabal run -- --csv " +``` + +## Process benchmark results + +Store the benchmark results with step using the action. + +```yaml +- name: Store benchmark result + uses: benchmark-action/github-action-benchmark@v1 + with: + tool: 'cabal' + output-file-path: +``` + +Please read ['How to use' section](https://github.com/benchmark-action/github-action-benchmark#how-to-use) for common usage. diff --git a/examples/cabal/fibber.cabal b/examples/cabal/fibber.cabal new file mode 100644 index 000000000..e5ee2bc9a --- /dev/null +++ b/examples/cabal/fibber.cabal @@ -0,0 +1,40 @@ +name: fibber +version: 0 +synopsis: Examples for the criterion benchmarking system +description: Examples for the criterion benchmarking system. +homepage: https://github.com/haskell/criterion +license: BSD3 +license-file: LICENSE +author: Bryan O'Sullivan +maintainer: Bryan O'Sullivan +category: Benchmarks +build-type: Simple +cabal-version: >=1.10 +tested-with: + GHC==7.4.2, + GHC==7.6.3, + GHC==7.8.4, + GHC==7.10.3, + GHC==8.0.2, + GHC==8.2.2, + GHC==8.4.4, + GHC==8.6.5, + GHC==8.8.4, + GHC==8.10.7, + GHC==9.0.2, + GHC==9.2.2 + +flag conduit-vs-pipes + default: True + +flag maps + default: True + +executable fibber + main-is: Fibber.hs + + default-language: Haskell2010 + ghc-options: -Wall -rtsopts + build-depends: + base == 4.*, + criterion diff --git a/src/config.ts b/src/config.ts index d6cf537d9..677b014c3 100644 --- a/src/config.ts +++ b/src/config.ts @@ -35,6 +35,7 @@ export const VALID_TOOLS = [ 'catch2', 'julia', 'benchmarkdotnet', + 'cabal', 'customBiggerIsBetter', 'customSmallerIsBetter', ] as const; diff --git a/src/default_index_html.ts b/src/default_index_html.ts index 142f135d5..d348ad654 100644 --- a/src/default_index_html.ts +++ b/src/default_index_html.ts @@ -122,6 +122,7 @@ export const DEFAULT_INDEX_HTML = String.raw` benchmarkdotnet: '#178600', customBiggerIsBetter: '#38ff38', customSmallerIsBetter: '#ff3838', + cabal: '#5e5086' _: '#333333' }; diff --git a/src/extract.ts b/src/extract.ts index 49e4349e2..b9498ab8a 100644 --- a/src/extract.ts +++ b/src/extract.ts @@ -614,6 +614,32 @@ function extractLuauBenchmarkResult(output: string): BenchmarkResult[] { return results; } +function extractCabalBenchmarkResult(output: string): BenchmarkResult[] { + const lines0 = output.split(/\n/); + const lines = lines0.splice(0, 1); + const res = []; + + for (const line of lines) { + const [name, meanSec, meanLBSec, meanUBSec, stddevSec, stddevLBSec, stddevUBSec] = line.split(','); + const [mean, meanUnit] = getHumanReadableUnitValue(parseFloat(meanSec)); + const [meanLB, meanLBUnit] = getHumanReadableUnitValue(parseFloat(meanLBSec)); + const [meanUB, meanUBUnit] = getHumanReadableUnitValue(parseFloat(meanUBSec)); + const [stddev, stddevUnit] = getHumanReadableUnitValue(parseFloat(stddevSec)); + const [stddevLB, stddevLBUnit] = getHumanReadableUnitValue(parseFloat(stddevLBSec)); + const [stddevUB, stddevUBUnit] = getHumanReadableUnitValue(parseFloat(stddevUBSec)); + + res.push({ + name: name, + value: mean, + unit: meanUnit, + range: `±${stddev} ${stddevUnit}`, + extra: `Mean lower bound: ${meanLB} ${meanLBUnit}\nMean upper bound: ${meanUB} ${meanUBUnit}\nStandard deviation LB: ${stddevLB} ${stddevLBUnit}\nStandard deviation UB: ${stddevUB} ${stddevUBUnit}`, + }); + } + + return res; +} + export async function extractResult(config: Config): Promise { const output = await fs.readFile(config.outputFilePath, 'utf8'); const { tool, githubToken } = config; @@ -644,6 +670,9 @@ export async function extractResult(config: Config): Promise { case 'benchmarkdotnet': benches = extractBenchmarkDotnetResult(output); break; + case 'cabal': + benches = extractCabalBenchmarkResult(output); + break; case 'customBiggerIsBetter': benches = extractCustomBenchmarkResult(output); break; diff --git a/src/write.ts b/src/write.ts index d7a60f779..4dd935eaf 100644 --- a/src/write.ts +++ b/src/write.ts @@ -77,6 +77,8 @@ function biggerIsBetter(tool: ToolType): boolean { return false; case 'benchmarkdotnet': return false; + case 'cabal': + return false; case 'customBiggerIsBetter': return true; case 'customSmallerIsBetter': diff --git a/test/data/extract/cabal_output.txt b/test/data/extract/cabal_output.txt new file mode 100644 index 000000000..32b2bc12f --- /dev/null +++ b/test/data/extract/cabal_output.txt @@ -0,0 +1,3 @@ +Name,Mean,MeanLB,MeanUB,Stddev,StddevLB,StddevUB +fib/1,1.2128410840686445e-8,1.2074536540005714e-8,1.2202216975790019e-8,2.145268159803889e-10,1.6198043933498568e-10,2.7812295435222917e-10 +fib/5,2.3776579995877364e-7,2.343297169016553e-7,2.4262841729253593e-7,1.3331560091086646e-8,9.007202760821372e-9,1.9969513691856127e-8 diff --git a/test/extract.spec.ts b/test/extract.spec.ts index 1532d1af0..489e3d272 100644 --- a/test/extract.spec.ts +++ b/test/extract.spec.ts @@ -417,6 +417,25 @@ describe('extractResult()', function () { }, ], }, + { + tool: 'cabal', + expected: [ + { + name: 'fib/1', + unit: 'nsec', + value: 12.128410840686445, + range: '±0.2145268159803889 nsec', + extra: 'Mean lower bound: 12.074536540005713 nsec\nMean upper bound: 12.20221697579002 nsec\nStandard deviation LB: 0.16198043933498568 nsec\nStandard deviation UB: 0.27812295435222917 nsec', + }, + { + name: 'fib/5', + unit: 'nsec', + value: 237.76579995877364, + range: '±13.331560091086645 nsec', + extra: 'Mean lower bound: 234.3297169016553 nsec\nMean upper bound: 242.62841729253594 nsec\nStandard deviation LB: 9.007202760821372 nsec\nStandard deviation UB: 19.96951369185613 nsec', + }, + ], + }, ]; for (const test of normalCases) {