Merge pull request #73 from awadell1/filter

breaking: Support filtering benchmarks and profiling

.github/workflows/CI.yml

@@ -19,8 +19,8 @@ jobs:
         arch:
           - x64
     steps:
-      - uses: actions/checkout@v3
-      - uses: julia-actions/setup-julia@v1
+      - uses: actions/checkout@v4
+      - uses: julia-actions/setup-julia@v2
         with:
           version: ${{ matrix.version }}
           arch: ${{ matrix.arch }}
@@ -32,11 +32,25 @@ jobs:
         env:
           COVERALLS_TOKEN: ${{ secrets.GITHUB_TOKEN }}
 
+  legacy:
+    name: Test Min Compat Bounds
+    runs-on: ubuntu-latest
+    steps:
+    - uses: julia-actions/julia-downgrade-compat@v1
+      if: ${{ matrix.os == 'ubuntu-latest' }}
+    - uses: actions/checkout@v4
+    - uses: julia-actions/setup-julia@v2
+      with:
+        version: 1.9
+    - uses: julia-actions/cache@v1
+    - uses: julia-actions/julia-buildpkg@v1
+    - uses: julia-actions/julia-runtest@v1
+
   action:
     name: Test Github action
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - uses: julia-actions/setup-julia@v1
       - uses: julia-actions/cache@v1
       - uses: "./"

Project.toml

@@ -1,7 +1,7 @@
 name = "PkgJogger"
 uuid = "10150987-6cc1-4b76-abee-b1c1cbd91c01"
 authors = ["Alexius Wadell <[email protected]> and contributors"]
-version = "0.5.1"
+version = "0.6.0"
 
 [deps]
 BSON = "fbb218c0-5317-5bc6-957e-2ee96dd4b1f0"
@@ -11,21 +11,38 @@ Dates = "ade2ca70-3891-5945-98fb-dc099432e06a"
 JSON = "682c06a0-de6a-54ab-a142-c8b1cf79cde6"
 LibGit2 = "76f85450-5226-5b5a-8eaa-529ad045b433"
 Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f"
+Profile = "9abbd945-dff8-562f-b5e8-e1ebf5ef1b79"
 Statistics = "10745b16-79ce-11e8-11f9-7d13ad32a3b2"
 Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
 UUIDs = "cf7118a7-6976-5b1a-9a39-7adc72f591a4"
 
+[weakdeps]
+CUDA = "052768ef-5323-5732-b1bb-66c8b64840ba"
+NVTX = "5da4648a-3479-48b8-97b9-01cb529c0a1f"
+
+[extensions]
+PkgJoggerCUDAExt = ["CUDA", "NVTX"]
+
 [compat]
 BSON = "0.3"
 BenchmarkTools = "1.5"
+CUDA = "5"
 CodecZlib = "0.7"
+Dates = "1.9"
 JSON = "0.21"
-Pkg = "1.9"
+LibGit2 = "1.9"
 Revise = "3"
+NVTX = "0.3"
+Pkg = "1.9"
+Profile = "1.9"
 Statistics = "1.9"
+Test = "1.9"
+UUIDs = "1.9"
 julia = "1.9"
 
 [extras]
+CUDA = "052768ef-5323-5732-b1bb-66c8b64840ba"
+NVTX = "5da4648a-3479-48b8-97b9-01cb529c0a1f"
 Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f"
 Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c"
 ReTestItems = "817f1d60-ba6b-4fd5-9520-3cf149f6a823"
@@ -35,4 +52,4 @@ Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
 UUIDs = "cf7118a7-6976-5b1a-9a39-7adc72f591a4"
 
 [targets]
-test = ["Test", "ReTestItems", "Revise", "UUIDs", "TOML", "Random", "Pkg"]
+test = ["Test", "ReTestItems", "Revise", "UUIDs", "TOML", "Random", "Pkg", "NVTX", "CUDA"]

docs/Project.toml

@@ -1,8 +1,14 @@
 [deps]
 BenchmarkTools = "6e4b80f9-dd63-53aa-95a3-0cdb28fa8baf"
+CUDA = "052768ef-5323-5732-b1bb-66c8b64840ba"
 Documenter = "e30172f5-a6a5-5a46-863b-614d45cd2de4"
+DocumenterInterLinks = "d12716ef-a0f6-4df4-a9f1-a5a34e75c656"
 Example = "4b09cd0b-9172-4840-a79f-b48550c7f881"
 PkgJogger = "10150987-6cc1-4b76-abee-b1c1cbd91c01"
+Profile = "9abbd945-dff8-562f-b5e8-e1ebf5ef1b79"
 
 [compat]
-Documenter = "1.2"
+Documenter = "1.4"
+
+[preferences.CUDA_Runtime_jll]
+local = true

docs/make.jl

@@ -10,7 +10,10 @@ Pkg.instantiate()
 
 # Load Code
 using Documenter
+using Documenter.Remotes: GitHub
+using DocumenterInterLinks
 using PkgJogger
+using CUDA
 using Example
 @jog Example
 
@@ -20,32 +23,47 @@ DocMeta.setdocmeta!(PkgJogger, :DocTestSetup, :(using PkgJogger); recursive=true
 index_md = joinpath(@__DIR__, "src", "index.md")
 readme_md = joinpath(@__DIR__, "..", "README.md")
 open(index_md, "w") do io
-    write(io, """
-    ```@meta
-    EditURL = "$readme_md"
-    ```
-    """)
+    write(
+        io,
+        """
+```@meta
+EditURL = "$readme_md"
+```
+"""
+    )
     write(io, read(readme_md, String))
 end
 
+# Interproject Links
+links = InterLinks(
+    "Julia" => "https://docs.julialang.org/en/v1/",
+    "CUDA" => "https://cuda.juliagpu.org/stable/",
+)
+
 makedocs(;
-    modules=[PkgJogger, JogExample],
+    modules=[
+        PkgJogger,
+        JogExample,
+        Base.get_extension(PkgJogger, :PkgJoggerCUDAExt),
+    ],
     authors="Alexius Wadell <[email protected]> and contributors",
-    repo="https://github.com/awadell1/PkgJogger.jl/blob/{commit}{path}#{line}",
+    repo=GitHub("awadell1", "PkgJogger.jl"),
     sitename="PkgJogger.jl",
     format=Documenter.HTML(;
         prettyurls=get(ENV, "CI", "false") == "true",
         canonical="https://awadell1.github.io/PkgJogger.jl",
         assets=String[],
-        analytics = "G-V9E0Q8BDHR",
+        analytics="G-V9E0Q8BDHR",
     ),
     pages=[
         "Home" => "index.md",
         "Jogger" => "jogger.md",
         "Saving Results" => "io.md",
         "Continuous Benchmarking" => "ci.md",
+        "Profiling" => "profiling.md",
         "Reference" => "reference.md",
     ],
+    plugins=[links],
     checkdocs=:all,
 )
 

docs/src/.gitignore

@@ -0,0 +1 @@
+index.md

docs/src/jogger.md

@@ -83,6 +83,20 @@ would have keys of `["bench_subdir", "bench_filename.jl", ...]`, instead of
 > run the suite of a single file, you can `include` the file and run it with:
 > `tune!(suite); run(suite)`
 
+## Filtering Benchmarks
+Often it's useful to run only a subset of the full benchmarking suite.
+For example, to run only the benchmarks within `benchmark/bench_filename.jl`:
+`JogExample.run("bench_filename")`.
+
+```julia
+using PkgJogger, Example
+@jog Example
+JogExample.run("bench_filename.jl")
+```
+
+!!! compat PkgJogger 0.6.0
+    Support for filtering via [`JogExample.suite`](@ref) was added in v0.6.0
+
 ## Jogger Reference
 
 Jogger modules provide helper methods for working with their package's
@@ -93,6 +107,7 @@ Example`.
 JogExample.suite
 JogExample.benchmark
 JogExample.run
+JogExample.profile
 JogExample.save_benchmarks
 JogExample.load_benchmarks
 JogExample.judge

docs/src/profiling.md

@@ -0,0 +1,26 @@
+# Profiling Benchmarks
+
+PkgJogger has support for profiling existing benchmarks using one of the [Supported Profilers](#supported-profilers).
+Profiling support is currently limited; notably:
+
+1. Only a single benchmark can be profiled at a time
+2. Automated saving or loading of profile results is not supported
+
+## Supported Profilers
+
+### CPU
+```@docs
+PkgJogger.profile(::Val{:cpu}, ::Any, ::PkgJogger.BenchmarkTools.Benchmark)
+```
+
+### Allocations
+
+```@docs
+PkgJogger.profile(::Val{:allocs}, ::Any, ::PkgJogger.BenchmarkTools.Benchmark)
+```
+
+### GPU
+
+```@docs
+PkgJogger.profile(::Val{:cuda}, ::Any, ::PkgJogger.BenchmarkTools.Benchmark)
+```

docs/src/reference.md

@@ -6,6 +6,7 @@ PkgJogger.locate_benchmarks
 PkgJogger.judge
 PkgJogger.test_benchmarks
 PkgJogger.tune!
+PkgJogger.getsuite
 ```
 
 ## Internal

ext/PkgJoggerCUDAExt.jl

@@ -0,0 +1,26 @@
+module PkgJoggerCUDAExt
+
+using PkgJogger
+using CUDA
+using NVTX
+
+"""
+    profiler=:cuda
+
+Profiles the benchmark using [`CUDA.@profile`](@extref).
+
+!!! warning
+    This only activates the CUDA profiler, you need to launch the profiler externally.
+    See [CUDA Profiling](@extref CUDA development/profiling) for documentation.
+
+"""
+function PkgJogger.profile(::Val{Symbol(:cuda)}, id, b::PkgJogger.BenchmarkTools.Benchmark; verbose)
+    id_str = join(id, "/")
+    CUDA.@profile external=true begin
+        NVTX.@range id_str begin
+            PkgJogger.BenchmarkTools.run(b)
+        end
+    end
+end
+
+end

src/PkgJogger.jl

@@ -10,6 +10,7 @@ using Dates
 using LibGit2
 using Statistics
 using Test
+using Profile
 
 export @jog, @test_benchmarks
 
@@ -30,6 +31,7 @@ const PKG_JOGGER_VER = VersionNumber(
 )
 
 include("utils.jl")
+include("profile.jl")
 include("jogger.jl")
 include("ci.jl")