Skip to content

Commit

Permalink
breaking: Add support for running a subset of the suite
Browse files Browse the repository at this point in the history 
Can now do `JogAwesome.benchmark("bench_foo.jl")` to only run
benchmarks within `bench_foo.jl`.

The downside is that specifying arguments to `BenchmarkTools.run` is
no longer possible. That is a breaking change, but the lost is the
ability to run with different parameters. So hopefully minor.
  • Loading branch information
@awadell1
awadell1 committed May 5, 2024
1 parent 4cee774 commit c54672d
Show file tree
Hide file tree
Showing 3 changed files with 228 additions and 120 deletions.
1 change: 1 addition & 0 deletions docs/src/reference.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ PkgJogger.locate_benchmarks
PkgJogger.judge
PkgJogger.test_benchmarks
PkgJogger.tune!
PkgJogger.getsuite
```

## Internal
Expand Down
310 changes: 191 additions & 119 deletions src/jogger.jl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,12 @@ using AwesomePkg, PkgJogger
results = JogAwesomePkg.benchmark()
file = JogAwesomePkg.save_benchmarks(results)
```
Compare benchmarking results to the latest saved results
```julia
results = JogAwesomePkg.benchmark()
JogAwesomePkg.judge(results, :latest)
```
"""
macro jog(pkg)
# Module Name
Expand All @@ -41,7 +47,6 @@ macro jog(pkg)
if !isdir(bench_dir)
error("No benchmark directory found for $pkg. Expected: $bench_dir")
end

# Generate Using Statements
using_statements = Expr[]
for pkg in JOGGER_PKGS
Expand All @@ -68,150 +73,217 @@ macro jog(pkg)
# Generate Module for Jogging pkg
quote
@eval module $modname
using $pkg
$(using_statements...)
using $pkg
$(using_statements...)

# Set Revise Mode and put submodules here
__revise_mode__ = :eval
$(suite_exp...)
# Set Revise Mode and put submodules here
__revise_mode__ = :eval
$(suite_exp...)

"""
BENCHMARK_DIR
"""
BENCHMARK_DIR
Directory of benchmarks for $($pkg)
"""
const BENCHMARK_DIR = $bench_dir
Directory of where benchmarking results are saved for $($pkg)
"""
const BENCHMARK_DIR = $bench_dir

"""
suite()::BenchmarkGroup
"""
suite()::BenchmarkGroup
The BenchmarkTools suite for $($pkg)
"""
function suite()
suite = BenchmarkTools.BenchmarkGroup()
$(suite_expressions...)
suite
end
The BenchmarkTools suite for $($pkg)
"""
function suite()
suite = BenchmarkTools.BenchmarkGroup()
$(suite_expressions...)
suite
end

# Dispatch calls to tune! here so we can use the jogger variant of load_benchmarks
__tune!(group::BenchmarkTools.BenchmarkGroup, ref::BenchmarkTools.BenchmarkGroup; kwargs...) = PkgJogger.tune!(group, ref; kwargs...)
__tune!(group::BenchmarkTools.BenchmarkGroup, ref; kwargs...) = PkgJogger.tune!(group, load_benchmarks(ref); kwargs...)
__tune!(group::BenchmarkTools.BenchmarkGroup, ::Nothing; kwargs...) = BenchmarkTools.tune!(group; kwargs...)

"""
benchmark(; verbose = false, save = false, ref = nothing)
Warmup, tune and run the benchmarking suite for $($pkg).
If `save = true`, will save the results using [`$($mod_str).save_benchmarks`](@ref)
and display the filename using `@info`.
To reuse prior tuning results set `ref` to a BenchmarkGroup or suitable identifier
for [`$($mod_str).load_benchmarks`](@ref). See [`PkgJogger.tune!`](@ref) for
more information about re-using tuning results.
"""
function benchmark(; verbose = false, save = false, ref = nothing)
s = suite()
__tune!(s, ref; verbose = verbose)
results = BenchmarkTools.run(s; verbose = verbose)
if save
filename = save_benchmarks(results)
@info "Saved results to $filename"
end
return results
end
"""
suite(select...)
"""
run(args...; verbose::Bool = false, kwargs)
Returns the benchmarking suite for $($pkg), optionally filtering based on `select...`.
At it's simplest, `$($mod_str).suite(a, b, ...)` is equivalent to `$($mod_str).suite()[a][b]...`
Run the benchmarking suite for $($pkg). See
[`BenchmarkTools.run`](https://juliaci.github.io/BenchmarkTools.jl/stable/reference/#Base.run)
for more options
"""
function run(args...; verbose = false, kwargs...)
BenchmarkTools.run(suite(), args...; verbose = verbose, kwargs...)
end
## Supported Indices
- `:` - Accepts any entry at that level in the tree
- `r"Regexp"` - Accepts any entry matching the regular-expression
- `key::Any` - Accepts any entry with a matching `key`
- `@tagged` - Filters the suite to only include `BenchmarkGroup`s with a matching tag.
See [Indexing into a BenchmarkGroup using @tagged](https://juliaci.github.io/BenchmarkTools.jl/stable/manual/#Indexing-into-a-BenchmarkGroup-using-@tagged)
"""
save_benchmarks(results::BenchmarkGroup)::String
!!! warning
An entry in `suite` must match all indices to be returned. For example,
`$($mod_str).suite(:, "bar")` would exclude a benchmark at `suite["bat"]` as
the benchmark isn't matched by **both** `:` and `"bar"`.
Saves benchmarking results for $($pkg) to `BENCHMARK_DIR/trial/uuid4().bson.gz`,
and returns the path to the saved results
## Examples
- The suite in `bench_foo.jl`: `$($mod_str).suite("bench_foo.jl")`
- Any benchmark matching `r"feature"` in any `bench_*.jl`: `$($mod_str).suite(:, r"feature")`
> Meta Data such as cpu load, time stamp, etc. are collected on save, not during
> benchmarking. For representative metadata, results should be saved immediately
> after benchmarking.
"""
suite(select...) = PkgJogger.getsuite(suite(), select...)

Results can be loaded with [`PkgJogger.load_benchmarks`](@ref) or
[`$($mod_str).load_benchmarks`](@ref)
# Dispatch calls to tune! here so we can use the jogger variant of load_benchmarks
__tune!(group::BenchmarkTools.BenchmarkGroup, ref::BenchmarkTools.BenchmarkGroup; kwargs...) = PkgJogger.tune!(group, ref; kwargs...)
__tune!(group::BenchmarkTools.BenchmarkGroup, ref; kwargs...) = PkgJogger.tune!(group, load_benchmarks(ref); kwargs...)
__tune!(group::BenchmarkTools.BenchmarkGroup, ::Nothing; kwargs...) = BenchmarkTools.tune!(group; kwargs...)

## Example
"""
benchmark([select...]; verbose = false, save = false, ref = nothing)
Running a benchmark suite and then saving the results
Warmup, tune and run the benchmarking suite for $($pkg).
```julia
r = $($mod_str).benchmark()
filename = $($mod_str).save_benchmarks(r)
```
If `save = true`, will save the results using [`$($mod_str).save_benchmarks`](@ref)
and display the filename using `@info`.
> Equivalently: `$($mod_str).benchmark(; save = true)`
To reuse prior tuning results set `ref` to a BenchmarkGroup or suitable identifier
for [`$($mod_str).load_benchmarks`](@ref). See [`PkgJogger.tune!`](@ref) for
more information about re-using tuning results.
"""
function save_benchmarks(results)
filename = joinpath(BENCHMARK_DIR, "trial", "$(UUIDs.uuid4()).bson.gz")
PkgJogger.save_benchmarks(filename, results)
filename
Optionally, benchmark a subset of the full suite by providing a set of filters.
See [`PkgJogger.getsuite`](@ref) for more information.
"""
function benchmark(select...; verbose=false, save=false, ref=nothing)
s = suite(select...)
BenchmarkTools.warmup(s; verbose)
__tune!(s, ref; verbose=verbose)
results = BenchmarkTools.run(s; verbose=verbose)
if save
filename = save_benchmarks(results)
@info "Saved results to $filename"
end
return results
end

"""
load_benchmarks(id)::Dict
"""
run([select...]; verbose::Bool = false, kwargs)
Loads benchmarking results for $($pkg) from `BENCHMARK_DIR/trial` based on `id`.
The following are supported `id` types:
Run the benchmarking suite for $($pkg). See
[`BenchmarkTools.run`](https://juliaci.github.io/BenchmarkTools.jl/stable/reference/#Base.run)
for more options
- `filename::String`: Loads results from `filename`
- `uuid::Union{String, UUID}`: Loads results with the given UUID
- `:latest` loads the latest (By mtime) results from `BENCHMARK_DIR/trial`
- `:oldest` loads the oldest (By mtime) results from `BENCHMARK_DIR/trial`
"""
load_benchmarks(id) = PkgJogger.load_benchmarks(joinpath(BENCHMARK_DIR, "trial"), id)
Optionally, run a subset of the full suite by providing a set of filters.
See [`PkgJogger.getsuite`](@ref) for more information.
"""
function run(select...; verbose=false, kwargs...)
BenchmarkTools.run(suite(select...); verbose=verbose, kwargs...)
end

"""
judge(new, old; metric=Statistics.median, kwargs...)
"""
save_benchmarks(results::BenchmarkGroup)::String
Compares benchmarking results from `new` vs `old` for regressions/improvements
using `metric` as a basis. Additional `kwargs` are passed to `BenchmarkTools.judge`
Saves benchmarking results for $($pkg) to `BENCHMARK_DIR/trial/uuid4().bson.gz`,
and returns the path to the saved results
Identical to [`PkgJogger.judge`](@ref), but accepts any identifier supported by
[`$($mod_str).load_benchmarks`](@ref)
> Meta Data such as cpu load, time stamp, etc. are collected on save, not during
> benchmarking. For representative metadata, results should be saved immediately
> after benchmarking.
## Examples
Results can be loaded with [`PkgJogger.load_benchmarks`](@ref) or
[`$($mod_str).load_benchmarks`](@ref)
```julia
# Judge the latest results vs. the oldest
$($mod_str).judge(:latest, :oldest)
[...]
```
## Examples
```julia
# Judge results by UUID
$($mod_str).judge("$(UUIDs.uuid4())", "$(UUIDs.uuid4())")
[...]
```
Running a benchmark suite and then saving the results
```julia
# Judge using the minimum, instead of the median, time
$($mod_str).judge("path/to/results.bson.gz", "$(UUIDs.uuid4())"; metric=minimum)
[...]
```
```julia
r = $($mod_str).benchmark()
filename = $($mod_str).save_benchmarks(r)
```
"""
function judge(new, old; kwargs...)
PkgJogger.judge(_get_benchmarks(new), _get_benchmarks(old); kwargs...)
end
_get_benchmarks(b) = load_benchmarks(b)
_get_benchmarks(b::Dict) = PkgJogger._get_benchmarks(b)
_get_benchmarks(b::BenchmarkTools.BenchmarkGroup) = b
> Equivalently: `$($mod_str).benchmark(; save = true)`
"""
function save_benchmarks(results)
filename = joinpath(BENCHMARK_DIR, "trial", "$(UUIDs.uuid4()).bson.gz")
PkgJogger.save_benchmarks(filename, results)
filename
end

"""
load_benchmarks(id)::Dict
Loads benchmarking results for $($pkg) from `BENCHMARK_DIR/trial` based on `id`.
The following are supported `id` types:
- `filename::String`: Loads results from `filename`
- `uuid::Union{String, UUID}`: Loads results with the given UUID
- `:latest` loads the latest (By mtime) results from `BENCHMARK_DIR/trial`
- `:oldest` loads the oldest (By mtime) results from `BENCHMARK_DIR/trial`
"""
load_benchmarks(id) = PkgJogger.load_benchmarks(joinpath(BENCHMARK_DIR, "trial"), id)

"""
judge(new, old; metric=Statistics.median, kwargs...)
Compares benchmarking results from `new` vs `old` for regressions/improvements
using `metric` as a basis. Additional `kwargs` are passed to `BenchmarkTools.judge`
Identical to [`PkgJogger.judge`](@ref), but accepts any identifier supported by
[`$($mod_str).load_benchmarks`](@ref)
## Examples
```julia
# Judge the latest results vs. the oldest
$($mod_str).judge(:latest, :oldest)
[...]
```
```julia
# Judge results by UUID
$($mod_str).judge("$(UUIDs.uuid4())", "$(UUIDs.uuid4())")
[...]
```
```julia
# Judge using the minimum, instead of the median, time
$($mod_str).judge("path/to/results.bson.gz", "$(UUIDs.uuid4())"; metric=minimum)
[...]
```
"""
function judge(new, old; kwargs...)
PkgJogger.judge(_get_benchmarks(new), _get_benchmarks(old); kwargs...)
end
_get_benchmarks(b) = load_benchmarks(b)
_get_benchmarks(b::Dict) = PkgJogger._get_benchmarks(b)
_get_benchmarks(b::BenchmarkTools.BenchmarkGroup) = b

"""
profile(select...; profiler=:cpu, verbose=false, ref=nothing, kwargs...)
Profile the benchmarking suite using the given `profiler`, the benchmark is
warmed up, tuned and then ran under the profile.
Like [`$($mod_str).benchmark`](@ref), `ref` can be used to reuse the results
of a prior run during tuning.
Some profilers support additional keyword arguments, see below for details.
!!! info
At this time, `PkgJogger` only supports profiling a single benchmark
at a time. Automated saving is not supported.
# Available Profilers
The following profilers are currently supported. Additional profilers
are available via package extensions.
$(@doc PkgJogger.profile)
---
!!! info
This list was generated on jogger creation (`@jog $($pkg)`),
and my not reflect all loaded extensions. See [`PkgJogger.profile`](@ref)
or regenerate the jogger for additional information
"""
function profile(select...; profiler::Symbol=:cpu, kwargs...)
s = suite(select...)
PkgJogger.profile(s, profiler; kwargs...)
end

end
end
Expand All @@ -230,15 +302,15 @@ function build_module(s::BenchModule)
# benchmarking module. Otherwise, don't track changes.
revise_id = PkgId(UUID("295af30f-e4ad-537b-8983-00126c2a3abe"), "Revise")
if haskey(Base.loaded_modules, revise_id)
revise_exp = :( Base.loaded_modules[$revise_id].track($modname, $(s.filename)) )
revise_exp = :(Base.loaded_modules[$revise_id].track($modname, $(s.filename)))
else
revise_exp = :()
end

module_expr = quote
module $modname
__revise_mode__ = :eval
include($(s.filename))
__revise_mode__ = :eval
include($(s.filename))
end
$(revise_exp)
end
Expand Down
Loading

0 comments on commit c54672d

Please sign in to comment.