feat: add optional test_args/folder_list/file_list

Project.toml

@@ -4,6 +4,7 @@ authors = ["Tamas K. Papp <[email protected]>"]
 version = "0.6.0"
 
 [deps]
+Coverage = "a2441757-f6aa-5fb2-8edb-039e3f45d037"
 CoverageTools = "c36e975a-824b-4404-a568-ef97ca766997"
 Dates = "ade2ca70-3891-5945-98fb-dc099432e06a"
 DefaultApplication = "3f0dd361-4fe0-5fc6-8523-80b14ec94d85"

src/LocalCoverage.jl

@@ -1,6 +1,7 @@
 module LocalCoverage
 
 import CoverageTools
+using Coverage: process_folder, process_file
 using DocStringExtensions: SIGNATURES, FIELDS
 using PrettyTables: pretty_table, Highlighter
 import DefaultApplication
@@ -175,7 +176,7 @@
 
 Evaluate the coverage metrics for the given pkg.
 """
-function eval_coverage_metrics(coverage, package_dir)
+function eval_coverage_metrics(coverage, package_dir)::PackageCoverage
     files = map(coverage) do file
         @unpack coverage = file
         lines_tracked = count(!isnothing, coverage)
@@ -199,6 +200,11 @@
 The test execution step may be skipped by passing `run_test = false`, allowing an
 easier use in combination with other test packages.
 
+Coverage of subsets of tests/files can be generated by specifying the list of testsets
+to run along with corresponding lists of files/folders.
+If a test is run on files/folders that *are not* in the list, then those files will
+be shown as having 0% coverage.
+
 An lcov file is also produced in `Pkg.dir(pkg, \"$(COVDIR)\", \"$(LCOVINFO)\")`.
 
 See [`report_coverage`](@ref), [`clean_coverage`](@ref).
@@ -207,19 +213,36 @@
 
 Printing of the result can be controlled via `IOContext`.
 
+# Example
 ```julia
 cov = generate_coverage(pkg)
 show(IOContext(stdout, :print_gaps => true), cov) # print coverage gap information
 ```
-
 """
-function generate_coverage(pkg = nothing; run_test = true)
+function generate_coverage(pkg = nothing; 
+                           run_test = true, 
+                           test_args = [""], 
+                           folder_list = ["src"], 
+                           file_list = [])::PackageCoverage
     if run_test
-        isnothing(pkg) ? Pkg.test(; coverage = true) : Pkg.test(pkg; coverage = true)
+        if isnothing(pkg)
+            Pkg.test(; coverage = true, test_args = test_args)
+        else
+            Pkg.test(pkg; coverage = true, test_args = test_args)
+        end
     end
     package_dir = pkgdir(pkg)
     cd(package_dir) do
-        coverage = CoverageTools.process_folder()
+        # initialize empty vector of coverage data
+        coverage = Vector{CoverageTools.FileCoverage}()
+        # process folders (same as default if `folder_list` isn't provided)
+        for f in folder_list
+            append!(coverage, process_folder(f))
+        end
+        # process individual files
+        for f in file_list
+            push!(coverage, process_file(f))
+        end
         mkpath(COVDIR)
         tracefile = joinpath(COVDIR, LCOVINFO)
         CoverageTools.LCOV.writefile(tracefile, coverage)
@@ -280,8 +303,14 @@
     nothing
 end
 
-function html_coverage(pkg = nothing; open = false, dir = tempdir())
-    html_coverage(generate_coverage(pkg), open = open, dir = dir)
+function html_coverage(pkg = nothing; 
+                       open = false, 
+                       dir = tempdir(), 
+                       test_args = [""], 
+                       folder_list = ["src"], 
+                       file_list = [])
+    gen_cov() = generate_coverage(pkg; test_args = test_args, folder_list = folder_list, file_list = file_list)
+    html_coverage(gen_cov(), open = open, dir = dir)
 end
 
 """
@@ -321,8 +350,12 @@
     exit(was_target_met ? 0 : 1)
 end
 
-function report_coverage_and_exit(pkg = nothing; kwargs...)
-    coverage = generate_coverage(pkg)
+function report_coverage_and_exit(pkg = nothing;
+                                  test_args = [""], 
+                                  folder_list = ["src"], 
+                                  file_list = [],
+                                  kwargs...)
+    coverage = generate_coverage(pkg; test_args = test_args, folder_list = folder_list, file_list = file_list)
     report_coverage_and_exit(coverage; kwargs...)
 end
 

test/DummyPackage/src/DummyPackage.jl

@@ -1,10 +1,13 @@
 "A package for testing coverage generation."
 module DummyPackage
 
-export foo, bar
+export foo, bar, baz, qux, quux, corge, grault
 
 foo() = 42
 
 include("bar.jl")
+include("qux.jl")
+include("corge/grault.jl")
+include("corge/corge.jl")
 
 end

test/DummyPackage/src/corge/corge.jl

@@ -0,0 +1,2 @@
+# tested in testset 2
+corge() = "corge"

test/DummyPackage/src/corge/grault.jl

@@ -0,0 +1,2 @@
+# untested
+grault() = 42

test/DummyPackage/src/qux.jl

@@ -0,0 +1,5 @@
+# tested in testset 1
+qux() = 43
+
+# untested
+quux() = 44

test/DummyPackage/test/runtests.jl

@@ -2,3 +2,11 @@ using Test, DummyPackage
 
 @test foo() == 42
 @test bar() == "a fish"
+
+@testset "testset 1" begin
+    @test qux() == 43
+end
+
+@testset "testset 2" begin
+    @test corge() == "corge"
+end

test/runtests.jl

@@ -6,49 +6,84 @@ Pkg.activate("./DummyPackage/")
 const pkg = "DummyPackage"      # we test the package with a dummy created for this purpose
 
 table_header = r"Filename\s+.\s+Lines\s+.\s+Hit\s+.\s+Miss\s+.\s+%"
-table_line = r"(?<!\/|\\\\)src(\/|\\\\)DummyPackage.jl?\s+.\s+\d+\s+.\s+\d+\s+.\s+\d+\s+.\s+\d+%"
+table_line = r"(?<!\/|\\\\)src(\/|\\\\)[\w\/]+\.jl?\s+.\s+\d+\s+.\s+\d+\s+.\s+\d+\s+.\s+\d+%"
 table_footer = r"TOTAL\s+.\s+\d+\s+.\s+\d+\s+.\s+\d+\s+.\s+\d+%"
 
 covdir = normpath(joinpath(@__DIR__, "DummyPackage", "coverage"))
 
-clean_coverage(pkg)
-@test isdir(LocalCoverage.pkgdir(pkg))
-lcovtrace = joinpath(covdir, "lcov.info")
-@test !isfile(lcovtrace)
+function test_coverage(pkg; 
+                       run_test = true, 
+                       test_args = [""], 
+                       folder_list = ["src"], 
+                       file_list = [])
+    @info "Testing coverage for $pkg" test_args folder_list file_list
+    clean_coverage(pkg)
+    @test isdir(LocalCoverage.pkgdir(pkg))
+    lcovtrace = joinpath(covdir, "lcov.info")
+    @test !isfile(lcovtrace)
 
-cov = generate_coverage(pkg)
-
-xmltrace = joinpath(covdir,"lcov.xml")
-write_lcov_to_xml(xmltrace, lcovtrace)
-open(xmltrace, "r") do io
-    header = readline(io)
-    doctype = readline(io)
-    @test header == """<?xml version="1.0" encoding="UTF-8"?>"""
-    @test startswith(doctype, "<!DOCTYPE coverage")
-end
+    cov = generate_coverage(pkg; 
+                            run_test = run_test, 
+                            test_args = test_args, 
+                            folder_list = folder_list, 
+                            file_list = file_list)
 
+    xmltrace = joinpath(covdir,"lcov.xml")
+    write_lcov_to_xml(xmltrace, lcovtrace)
+    open(xmltrace, "r") do io
+        header = readline(io)
+        doctype = readline(io)
+        @test header == """<?xml version="1.0" encoding="UTF-8"?>"""
+        @test startswith(doctype, "<!DOCTYPE coverage")
+    end
 
-buffer = IOBuffer()
-show(buffer, cov)
-table = String(take!(buffer))
-println(table)
-@test !isnothing(match(table_header, table))
-@test !isnothing(match(table_line, table))
-@test !isnothing(match(table_footer, table))
+    buffer = IOBuffer()
+    show(buffer, cov)
+    table = String(take!(buffer))
+    println(table)
+    @test !isnothing(match(table_header, table))
+    @test !isnothing(match(table_line, table))
+    @test !isnothing(match(table_footer, table))
 
-if !isnothing(Sys.which("genhtml"))
-    mktempdir() do dir
-        html_coverage(pkg, dir = dir)
-        @test isfile(joinpath(dir, "index.html"))
+    if !isnothing(Sys.which("genhtml"))
+        mktempdir() do dir
+            html_coverage(pkg, dir = dir)
+            @test isfile(joinpath(dir, "index.html"))
+        end
     end
+
+    @test isfile(lcovtrace)
+    rm(covdir, recursive = true)
+
+    @info "Printing coverage information for visual debugging"
+    show(stdout, cov)
+    show(IOContext(stdout, :print_gaps => true), cov)
 end
 
-@test isfile(lcovtrace)
-rm(covdir, recursive = true)
+@testset verbose = true "Testing coverage with" begin
+    @testset "default values" begin
+        test_coverage("DummyPackage")
+    end
+
+    @testset "test_args and file_list" begin
+        test_coverage("DummyPackage"; 
+                    test_args = ["testset 2"], 
+                    file_list = [joinpath(dirname(@__FILE__), "DummyPackage", "src", "qux.jl")])
+    end
+
+    @testset "test_args and folder_list" begin
+        test_coverage("DummyPackage"; 
+                    test_args = ["testset 1"], 
+                    folder_list = [joinpath(dirname(@__FILE__), "DummyPackage", "src", "corge")])
+    end
 
-@info "Printing coverage infomation for visual debugging"
-show(stdout, cov)
-show(IOContext(stdout, :print_gaps => true), cov)
+    @testset "test_args and file_list and folder_list" begin
+        test_coverage("DummyPackage"; 
+                    test_args = ["testset 1", "testset 2"], 
+                    folder_list = [joinpath(dirname(@__FILE__), "DummyPackage", "src", "corge")],
+                    file_list = [joinpath(dirname(@__FILE__), "DummyPackage", "src", "qux.jl")])
+    end
+end
 
 @test LocalCoverage.find_gaps([nothing, 0, 0, 0, 2, 3, 0, nothing, 0, 3, 0, 6, 2]) ==
     [2:4, 7:7, 9:9, 11:11]