From 2532af8ad06fbe0d10243649bd12ac4c70d7b644 Mon Sep 17 00:00:00 2001 From: zapashcanon Date: Wed, 29 May 2024 17:22:48 +0200 Subject: [PATCH] add binary to perform a diff between two bench results --- .gitignore | 2 +- bench/README.md | 8 ++- bench/diff/diff.ml | 147 ++++++++++++++++++++++++++++++++++++++++++ bench/diff/dune | 4 ++ bench/report/runs.ml | 4 ++ bench/report/runs.mli | 6 ++ 6 files changed, 168 insertions(+), 3 deletions(-) create mode 100644 bench/diff/diff.ml create mode 100644 bench/diff/dune diff --git a/.gitignore b/.gitignore index 40e5ce3e4..178f65006 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,4 @@ _build _coverage owi-out -results-testcomp* +results-* diff --git a/bench/README.md b/bench/README.md index d23d690b6..e86b781ee 100644 --- a/bench/README.md +++ b/bench/README.md @@ -28,10 +28,14 @@ A folder `testcomp-results-YYYY-MM-DD_HHhMMhSSs` has been created with a lot of ## Generate the report by hand -### Running on some benchmark output - ```shell-session $ dune exec -- report/bin/report.exe testcomp/results # this is an example of an old run, use the correct file instead ``` A folder `results-report` should be available in the working directory with the `index.html` file that contains the results. + +## Comparing two runs + +```shell-session +$ dune exec diff/diff.exe -- results1 results2 +``` diff --git a/bench/diff/diff.ml b/bench/diff/diff.ml new file mode 100644 index 000000000..a1d61a852 --- /dev/null +++ b/bench/diff/diff.ml @@ -0,0 +1,147 @@ +let () = + if Array.length Sys.argv < 3 then + Format.ksprintf failwith "usage: %s " Sys.argv.(0) + +let file1 = Fpath.v Sys.argv.(1) + +let file2 = Fpath.v Sys.argv.(2) + +open Report + +let report1 = Parse.from_file file1 + +let report2 = Parse.from_file file2 + +let count_all1 = Runs.count_all report1 + +let count_all2 = Runs.count_all report2 + +let count_nothing1 = Runs.count_nothing report1 + +let count_nothing2 = Runs.count_nothing report2 + +let count_reached1 = Runs.count_reached report1 + +let count_reached2 = Runs.count_reached report2 + +let count_timeout1 = Runs.count_timeout report1 + +let count_timeout2 = Runs.count_timeout report2 + +let count_other1 = Runs.count_other report1 + +let count_other2 = Runs.count_other report2 + +let count_killed1 = Runs.count_killed report1 + +let count_killed2 = Runs.count_killed report2 + +let reached1 = Runs.keep_reached report1 + +let reached2 = Runs.keep_reached report2 + +let reached_files1 = Runs.files reached1 + +let reached_files2 = Runs.files reached2 + +let reached_tbl1 = + let tbl = Hashtbl.create 512 in + List.iter (fun file -> Hashtbl.replace tbl file ()) reached_files1; + tbl + +let reached_tbl2 = + let tbl = Hashtbl.create 512 in + List.iter (fun file -> Hashtbl.replace tbl file ()) reached_files2; + tbl + +let reached_common = + let tbl = Hashtbl.create 512 in + List.iter + (fun file -> + if Hashtbl.mem reached_tbl2 file then Hashtbl.replace tbl file () ) + reached_files1; + tbl + +let count_common = Hashtbl.length reached_common + +let reached_common_1 = + Runs.keep_if (fun run -> Hashtbl.mem reached_common run.Run.file) reached1 + +let reached_common_2 = + Runs.keep_if (fun run -> Hashtbl.mem reached_common run.Run.file) reached2 + +let reached_only_1 = + Runs.keep_if + (fun run -> not @@ Hashtbl.mem reached_common run.Run.file) + reached1 + +let reached_only_2 = + Runs.keep_if + (fun run -> not @@ Hashtbl.mem reached_common run.Run.file) + reached2 + +let report1_found_by_2_not_by_1 = + Runs.keep_if + (fun run -> + let f = run.Run.file in + Hashtbl.mem reached_tbl2 f && (not @@ Hashtbl.mem reached_tbl1 f) ) + report1 + +let report2_found_by_1_not_by_2 = + Runs.keep_if + (fun run -> + let f = run.Run.file in + Hashtbl.mem reached_tbl1 f && (not @@ Hashtbl.mem reached_tbl2 f) ) + report2 + +let () = + if count_all1 <> count_all2 then + Format.printf + "WARNING: runs don't have the same total of runs (tool 1 has %d and tool \ + 2 has %d)@\n" + count_all1 count_all2; + Format.printf "tool1 had %03d reached and tool2 had %03d reached@\n" + count_reached1 count_reached2; + Format.printf "tool1 had %03d timeout and tool2 had %03d timeout@\n" + count_timeout1 count_timeout2; + Format.printf "tool1 had %03d nothing and tool2 had %03d nothing@\n" + count_nothing1 count_nothing2; + Format.printf "tool1 had %03d other and tool2 had %03d other@\n" + count_other1 count_other2; + Format.printf "tool1 had %03d killed and tool2 had %03d killed@\n@\n" + count_killed1 count_killed2; + Format.printf "tools have %03d reached tasks in common@\n@\n" + (Hashtbl.length reached_common); + Format.printf "tool1 had %03d tasks tool2 did not found@\n" + (count_reached1 - count_common); + Format.printf "tool2 had %03d tasks tool1 did not found@\n@\n" + (count_reached2 - count_common); + Format.printf + "on commonly reached tasks, tool 1 took %04f sec. (mean %04f) and \ + tool 2 took %06f sec. (mean %04f)@\n" + (Runs.sum_clock reached_common_1) + (Runs.mean_clock reached_common_1) + (Runs.sum_clock reached_common_2) + (Runs.mean_clock reached_common_2); + Format.printf + "on *not commonly* reached tasks, tool 1 took %04f sec. (mean %04f) and \ + tool 2 took %06f sec. (mean %04f)@\n\ + @\n" + (Runs.sum_clock reached_only_1) + (Runs.mean_clock reached_only_1) + (Runs.sum_clock reached_only_2) + (Runs.mean_clock reached_only_2); + Format.printf + "among tasks reached only by tool 1, tool 2 replied %03d nothing, %03d \ + timeout, %02d other and %02d killed@\n" + (Runs.count_nothing report2_found_by_1_not_by_2) + (Runs.count_timeout report2_found_by_1_not_by_2) + (Runs.count_other report2_found_by_1_not_by_2) + (Runs.count_killed report2_found_by_1_not_by_2); + Format.printf + "among tasks reached only by tool 2, tool 1 replied %03d nothing, %03d \ + timeout, %02d other and %02d killed@\n" + (Runs.count_nothing report1_found_by_2_not_by_1) + (Runs.count_timeout report1_found_by_2_not_by_1) + (Runs.count_other report1_found_by_2_not_by_1) + (Runs.count_killed report1_found_by_2_not_by_1) diff --git a/bench/diff/dune b/bench/diff/dune new file mode 100644 index 000000000..40b88970d --- /dev/null +++ b/bench/diff/dune @@ -0,0 +1,4 @@ +(executable + (name diff) + (modules diff) + (libraries report tool)) diff --git a/bench/report/runs.ml b/bench/report/runs.ml index f73831384..92bd66963 100644 --- a/bench/report/runs.ml +++ b/bench/report/runs.ml @@ -41,6 +41,8 @@ let keep_other = List.filter Run.is_other let keep_killed = List.filter Run.is_killed +let keep_if f runs = List.filter f runs + let min_clock runs = match runs with | [] -> 0. @@ -98,3 +100,5 @@ let pp_quick_results fmt results = !nothing !reached !timeout !other !killed let map = List.map + +let files = List.map (fun run -> run.Run.file) diff --git a/bench/report/runs.mli b/bench/report/runs.mli index 2a5227793..884bff1a5 100644 --- a/bench/report/runs.mli +++ b/bench/report/runs.mli @@ -4,6 +4,8 @@ val empty : t val add : Run.t -> t -> t +val count_all : t -> int + val count_nothing : t -> int val count_reached : t -> int @@ -24,6 +26,8 @@ val keep_other : t -> t val keep_killed : t -> t +val keep_if : (Run.t -> bool) -> t -> t + val min_clock : t -> float val max_clock : t -> float @@ -45,3 +49,5 @@ val to_distribution : max_time:int -> t -> float list val pp_quick_results : Format.formatter -> t -> unit val map : (Run.t -> 'a) -> t -> 'a list + +val files : t -> Fpath.t list