From c54672d18307ef4a2e800e7380a58ecc38d1dafc Mon Sep 17 00:00:00 2001 From: Alexius Wadell Date: Wed, 16 Aug 2023 16:15:42 -0400 Subject: [PATCH] breaking: Add support for running a subset of the suite Can now do `JogAwesome.benchmark("bench_foo.jl")` to only run benchmarks within `bench_foo.jl`. The downside is that specifying arguments to `BenchmarkTools.run` is no longer possible. That is a breaking change, but the lost is the ability to run with different parameters. So hopefully minor. --- docs/src/reference.md | 1 + src/jogger.jl | 310 ++++++++++++++++++++++++++---------------- src/utils.jl | 37 ++++- 3 files changed, 228 insertions(+), 120 deletions(-) diff --git a/docs/src/reference.md b/docs/src/reference.md index 35abad3..fb71ac4 100644 --- a/docs/src/reference.md +++ b/docs/src/reference.md @@ -6,6 +6,7 @@ PkgJogger.locate_benchmarks PkgJogger.judge PkgJogger.test_benchmarks PkgJogger.tune! +PkgJogger.getsuite ``` ## Internal diff --git a/src/jogger.jl b/src/jogger.jl index 19c5fdb..0f9e8d4 100644 --- a/src/jogger.jl +++ b/src/jogger.jl @@ -31,6 +31,12 @@ using AwesomePkg, PkgJogger results = JogAwesomePkg.benchmark() file = JogAwesomePkg.save_benchmarks(results) ``` + +Compare benchmarking results to the latest saved results +```julia +results = JogAwesomePkg.benchmark() +JogAwesomePkg.judge(results, :latest) +``` """ macro jog(pkg) # Module Name @@ -41,7 +47,6 @@ macro jog(pkg) if !isdir(bench_dir) error("No benchmark directory found for $pkg. Expected: $bench_dir") end - # Generate Using Statements using_statements = Expr[] for pkg in JOGGER_PKGS @@ -68,150 +73,217 @@ macro jog(pkg) # Generate Module for Jogging pkg quote @eval module $modname - using $pkg - $(using_statements...) + using $pkg + $(using_statements...) - # Set Revise Mode and put submodules here - __revise_mode__ = :eval - $(suite_exp...) + # Set Revise Mode and put submodules here + __revise_mode__ = :eval + $(suite_exp...) - """ - BENCHMARK_DIR + """ + BENCHMARK_DIR - Directory of benchmarks for $($pkg) - """ - const BENCHMARK_DIR = $bench_dir + Directory of where benchmarking results are saved for $($pkg) + """ + const BENCHMARK_DIR = $bench_dir - """ - suite()::BenchmarkGroup + """ + suite()::BenchmarkGroup - The BenchmarkTools suite for $($pkg) - """ - function suite() - suite = BenchmarkTools.BenchmarkGroup() - $(suite_expressions...) - suite - end + The BenchmarkTools suite for $($pkg) + """ + function suite() + suite = BenchmarkTools.BenchmarkGroup() + $(suite_expressions...) + suite + end - # Dispatch calls to tune! here so we can use the jogger variant of load_benchmarks - __tune!(group::BenchmarkTools.BenchmarkGroup, ref::BenchmarkTools.BenchmarkGroup; kwargs...) = PkgJogger.tune!(group, ref; kwargs...) - __tune!(group::BenchmarkTools.BenchmarkGroup, ref; kwargs...) = PkgJogger.tune!(group, load_benchmarks(ref); kwargs...) - __tune!(group::BenchmarkTools.BenchmarkGroup, ::Nothing; kwargs...) = BenchmarkTools.tune!(group; kwargs...) - - """ - benchmark(; verbose = false, save = false, ref = nothing) - - Warmup, tune and run the benchmarking suite for $($pkg). - - If `save = true`, will save the results using [`$($mod_str).save_benchmarks`](@ref) - and display the filename using `@info`. - - To reuse prior tuning results set `ref` to a BenchmarkGroup or suitable identifier - for [`$($mod_str).load_benchmarks`](@ref). See [`PkgJogger.tune!`](@ref) for - more information about re-using tuning results. - """ - function benchmark(; verbose = false, save = false, ref = nothing) - s = suite() - __tune!(s, ref; verbose = verbose) - results = BenchmarkTools.run(s; verbose = verbose) - if save - filename = save_benchmarks(results) - @info "Saved results to $filename" - end - return results - end + """ + suite(select...) - """ - run(args...; verbose::Bool = false, kwargs) + Returns the benchmarking suite for $($pkg), optionally filtering based on `select...`. + At it's simplest, `$($mod_str).suite(a, b, ...)` is equivalent to `$($mod_str).suite()[a][b]...` - Run the benchmarking suite for $($pkg). See - [`BenchmarkTools.run`](https://juliaci.github.io/BenchmarkTools.jl/stable/reference/#Base.run) - for more options - """ - function run(args...; verbose = false, kwargs...) - BenchmarkTools.run(suite(), args...; verbose = verbose, kwargs...) - end + ## Supported Indices + + - `:` - Accepts any entry at that level in the tree + - `r"Regexp"` - Accepts any entry matching the regular-expression + - `key::Any` - Accepts any entry with a matching `key` + - `@tagged` - Filters the suite to only include `BenchmarkGroup`s with a matching tag. + See [Indexing into a BenchmarkGroup using @tagged](https://juliaci.github.io/BenchmarkTools.jl/stable/manual/#Indexing-into-a-BenchmarkGroup-using-@tagged) - """ - save_benchmarks(results::BenchmarkGroup)::String + !!! warning + An entry in `suite` must match all indices to be returned. For example, + `$($mod_str).suite(:, "bar")` would exclude a benchmark at `suite["bat"]` as + the benchmark isn't matched by **both** `:` and `"bar"`. - Saves benchmarking results for $($pkg) to `BENCHMARK_DIR/trial/uuid4().bson.gz`, - and returns the path to the saved results + ## Examples + - The suite in `bench_foo.jl`: `$($mod_str).suite("bench_foo.jl")` + - Any benchmark matching `r"feature"` in any `bench_*.jl`: `$($mod_str).suite(:, r"feature")` - > Meta Data such as cpu load, time stamp, etc. are collected on save, not during - > benchmarking. For representative metadata, results should be saved immediately - > after benchmarking. + """ + suite(select...) = PkgJogger.getsuite(suite(), select...) - Results can be loaded with [`PkgJogger.load_benchmarks`](@ref) or - [`$($mod_str).load_benchmarks`](@ref) + # Dispatch calls to tune! here so we can use the jogger variant of load_benchmarks + __tune!(group::BenchmarkTools.BenchmarkGroup, ref::BenchmarkTools.BenchmarkGroup; kwargs...) = PkgJogger.tune!(group, ref; kwargs...) + __tune!(group::BenchmarkTools.BenchmarkGroup, ref; kwargs...) = PkgJogger.tune!(group, load_benchmarks(ref); kwargs...) + __tune!(group::BenchmarkTools.BenchmarkGroup, ::Nothing; kwargs...) = BenchmarkTools.tune!(group; kwargs...) - ## Example + """ + benchmark([select...]; verbose = false, save = false, ref = nothing) - Running a benchmark suite and then saving the results + Warmup, tune and run the benchmarking suite for $($pkg). - ```julia - r = $($mod_str).benchmark() - filename = $($mod_str).save_benchmarks(r) - ``` + If `save = true`, will save the results using [`$($mod_str).save_benchmarks`](@ref) + and display the filename using `@info`. - > Equivalently: `$($mod_str).benchmark(; save = true)` + To reuse prior tuning results set `ref` to a BenchmarkGroup or suitable identifier + for [`$($mod_str).load_benchmarks`](@ref). See [`PkgJogger.tune!`](@ref) for + more information about re-using tuning results. - """ - function save_benchmarks(results) - filename = joinpath(BENCHMARK_DIR, "trial", "$(UUIDs.uuid4()).bson.gz") - PkgJogger.save_benchmarks(filename, results) - filename + Optionally, benchmark a subset of the full suite by providing a set of filters. + See [`PkgJogger.getsuite`](@ref) for more information. + """ + function benchmark(select...; verbose=false, save=false, ref=nothing) + s = suite(select...) + BenchmarkTools.warmup(s; verbose) + __tune!(s, ref; verbose=verbose) + results = BenchmarkTools.run(s; verbose=verbose) + if save + filename = save_benchmarks(results) + @info "Saved results to $filename" end + return results + end - """ - load_benchmarks(id)::Dict + """ + run([select...]; verbose::Bool = false, kwargs) - Loads benchmarking results for $($pkg) from `BENCHMARK_DIR/trial` based on `id`. - The following are supported `id` types: + Run the benchmarking suite for $($pkg). See + [`BenchmarkTools.run`](https://juliaci.github.io/BenchmarkTools.jl/stable/reference/#Base.run) + for more options - - `filename::String`: Loads results from `filename` - - `uuid::Union{String, UUID}`: Loads results with the given UUID - - `:latest` loads the latest (By mtime) results from `BENCHMARK_DIR/trial` - - `:oldest` loads the oldest (By mtime) results from `BENCHMARK_DIR/trial` - """ - load_benchmarks(id) = PkgJogger.load_benchmarks(joinpath(BENCHMARK_DIR, "trial"), id) + Optionally, run a subset of the full suite by providing a set of filters. + See [`PkgJogger.getsuite`](@ref) for more information. + """ + function run(select...; verbose=false, kwargs...) + BenchmarkTools.run(suite(select...); verbose=verbose, kwargs...) + end - """ - judge(new, old; metric=Statistics.median, kwargs...) + """ + save_benchmarks(results::BenchmarkGroup)::String - Compares benchmarking results from `new` vs `old` for regressions/improvements - using `metric` as a basis. Additional `kwargs` are passed to `BenchmarkTools.judge` + Saves benchmarking results for $($pkg) to `BENCHMARK_DIR/trial/uuid4().bson.gz`, + and returns the path to the saved results - Identical to [`PkgJogger.judge`](@ref), but accepts any identifier supported by - [`$($mod_str).load_benchmarks`](@ref) + > Meta Data such as cpu load, time stamp, etc. are collected on save, not during + > benchmarking. For representative metadata, results should be saved immediately + > after benchmarking. - ## Examples + Results can be loaded with [`PkgJogger.load_benchmarks`](@ref) or + [`$($mod_str).load_benchmarks`](@ref) - ```julia - # Judge the latest results vs. the oldest - $($mod_str).judge(:latest, :oldest) - [...] - ``` + ## Examples - ```julia - # Judge results by UUID - $($mod_str).judge("$(UUIDs.uuid4())", "$(UUIDs.uuid4())") - [...] - ``` + Running a benchmark suite and then saving the results - ```julia - # Judge using the minimum, instead of the median, time - $($mod_str).judge("path/to/results.bson.gz", "$(UUIDs.uuid4())"; metric=minimum) - [...] - ``` + ```julia + r = $($mod_str).benchmark() + filename = $($mod_str).save_benchmarks(r) + ``` - """ - function judge(new, old; kwargs...) - PkgJogger.judge(_get_benchmarks(new), _get_benchmarks(old); kwargs...) - end - _get_benchmarks(b) = load_benchmarks(b) - _get_benchmarks(b::Dict) = PkgJogger._get_benchmarks(b) - _get_benchmarks(b::BenchmarkTools.BenchmarkGroup) = b + > Equivalently: `$($mod_str).benchmark(; save = true)` + + """ + function save_benchmarks(results) + filename = joinpath(BENCHMARK_DIR, "trial", "$(UUIDs.uuid4()).bson.gz") + PkgJogger.save_benchmarks(filename, results) + filename + end + + """ + load_benchmarks(id)::Dict + + Loads benchmarking results for $($pkg) from `BENCHMARK_DIR/trial` based on `id`. + The following are supported `id` types: + + - `filename::String`: Loads results from `filename` + - `uuid::Union{String, UUID}`: Loads results with the given UUID + - `:latest` loads the latest (By mtime) results from `BENCHMARK_DIR/trial` + - `:oldest` loads the oldest (By mtime) results from `BENCHMARK_DIR/trial` + """ + load_benchmarks(id) = PkgJogger.load_benchmarks(joinpath(BENCHMARK_DIR, "trial"), id) + + """ + judge(new, old; metric=Statistics.median, kwargs...) + + Compares benchmarking results from `new` vs `old` for regressions/improvements + using `metric` as a basis. Additional `kwargs` are passed to `BenchmarkTools.judge` + + Identical to [`PkgJogger.judge`](@ref), but accepts any identifier supported by + [`$($mod_str).load_benchmarks`](@ref) + + ## Examples + + ```julia + # Judge the latest results vs. the oldest + $($mod_str).judge(:latest, :oldest) + [...] + ``` + + ```julia + # Judge results by UUID + $($mod_str).judge("$(UUIDs.uuid4())", "$(UUIDs.uuid4())") + [...] + ``` + + ```julia + # Judge using the minimum, instead of the median, time + $($mod_str).judge("path/to/results.bson.gz", "$(UUIDs.uuid4())"; metric=minimum) + [...] + ``` + """ + function judge(new, old; kwargs...) + PkgJogger.judge(_get_benchmarks(new), _get_benchmarks(old); kwargs...) + end + _get_benchmarks(b) = load_benchmarks(b) + _get_benchmarks(b::Dict) = PkgJogger._get_benchmarks(b) + _get_benchmarks(b::BenchmarkTools.BenchmarkGroup) = b + + """ + profile(select...; profiler=:cpu, verbose=false, ref=nothing, kwargs...) + + Profile the benchmarking suite using the given `profiler`, the benchmark is + warmed up, tuned and then ran under the profile. + + Like [`$($mod_str).benchmark`](@ref), `ref` can be used to reuse the results + of a prior run during tuning. + + Some profilers support additional keyword arguments, see below for details. + + !!! info + At this time, `PkgJogger` only supports profiling a single benchmark + at a time. Automated saving is not supported. + + # Available Profilers + The following profilers are currently supported. Additional profilers + are available via package extensions. + + + $(@doc PkgJogger.profile) + + --- + + !!! info + This list was generated on jogger creation (`@jog $($pkg)`), + and my not reflect all loaded extensions. See [`PkgJogger.profile`](@ref) + or regenerate the jogger for additional information + + """ + function profile(select...; profiler::Symbol=:cpu, kwargs...) + s = suite(select...) + PkgJogger.profile(s, profiler; kwargs...) + end end end @@ -230,15 +302,15 @@ function build_module(s::BenchModule) # benchmarking module. Otherwise, don't track changes. revise_id = PkgId(UUID("295af30f-e4ad-537b-8983-00126c2a3abe"), "Revise") if haskey(Base.loaded_modules, revise_id) - revise_exp = :( Base.loaded_modules[$revise_id].track($modname, $(s.filename)) ) + revise_exp = :(Base.loaded_modules[$revise_id].track($modname, $(s.filename))) else revise_exp = :() end module_expr = quote module $modname - __revise_mode__ = :eval - include($(s.filename)) + __revise_mode__ = :eval + include($(s.filename)) end $(revise_exp) end diff --git a/src/utils.jl b/src/utils.jl index 0695499..80b5870 100644 --- a/src/utils.jl +++ b/src/utils.jl @@ -71,6 +71,41 @@ function locate_benchmarks(path, name=String[]) end locate_benchmarks(pkg::Module) = benchmark_dir(pkg) |> locate_benchmarks +""" + getsuite(suite, [select...]) + +Index into `suite` and return the matching entries in suite. +At it's simplest, `getsuite(suite, "foo", "bar",...)` is the same as `suite["foo"]["bar"]...` + +# Supported Indices + +- `:` - Accepts any entry at that level in the tree +- `r"Regexp"` - Accepts any entry matching the regular-expression +- `key::Any` - Accepts any entry with a matching `key` +- `@tagged` - Filters the suite to only include `BenchmarkGroup`s with a matching tag. + See [Indexing into a BenchmarkGroup using @tagged](https://juliaci.github.io/BenchmarkTools.jl/stable/manual/#Indexing-into-a-BenchmarkGroup-using-@tagged) + +!!! warning + An entry in `suite` must match all indices to be returned. For example, + `getsuite(s, :, "bar")` would exclude a benchmark at `s["bat"]` as + the benchmark isn't matched by **both** `:` and `"bar"`. +""" +getsuite(suite::BenchmarkGroup) = suite +getsuite(suite::BenchmarkGroup, ::Colon) = suite +getsuite(suite::BenchmarkGroup, r::Regex) = filter(!isnothing ∘ Base.Fix1(match, r) ∘ first, suite) +getsuite(suite::BenchmarkGroup, f::BenchmarkTools.TagFilter) = suite[f] +getsuite(::BenchmarkTools.Benchmark, ::Any) = nothing +getsuite(suite::BenchmarkGroup, idx) = !haskey(suite, idx) ? BenchmarkGroup() : BenchmarkGroup(idx => suite[idx]) +function getsuite(suite::BenchmarkGroup, idx, rest...) + src = getsuite(suite, idx) + dst = similar(src) + for (k, v) in src + v = getsuite(v, rest...) + !isnothing(v) && !isempty(v) && setindex!(dst, v, k) + end + return dst +end + """ judge(new, old; metric=Statistics.median, kwargs...) @@ -87,7 +122,7 @@ Effectively a convenience wrapper around `load_benchmarks` and `BenchmarkTools.j function judge( new::BenchmarkTools.BenchmarkGroup, old::BenchmarkTools.BenchmarkGroup; - metric = Statistics.median, + metric=Statistics.median, kwargs... ) new_estimate = metric(new)