From 73eb025593958c64a755888761951fcbfa55764e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aurimas=20Bla=C5=BEulionis?= <0x60@pm.me> Date: Mon, 26 Aug 2019 15:57:36 -0700 Subject: [PATCH] Adds ability to plot throughput on summary page. Fixes #149. --- src/html/mod.rs | 27 +++++++++++-- src/html/summary_report.html.tt | 7 +++- src/lib.rs | 2 +- src/plot/gnuplot_backend/mod.rs | 6 ++- src/plot/gnuplot_backend/summary.rs | 29 ++++++++------ src/plot/mod.rs | 57 ++++++++++++++++++++++++++++ src/plot/plotters_backend/mod.rs | 6 ++- src/plot/plotters_backend/summary.rs | 45 ++++++++++++++-------- 8 files changed, 143 insertions(+), 36 deletions(-) diff --git a/src/html/mod.rs b/src/html/mod.rs index eb31a8168..822674df9 100644 --- a/src/html/mod.rs +++ b/src/html/mod.rs @@ -5,7 +5,7 @@ use crate::estimate::Estimate; use crate::format; use crate::fs; use crate::measurement::ValueFormatter; -use crate::plot::{PlotContext, PlotData, Plotter}; +use crate::plot::{LinePlotConfig, PlotContext, PlotData, Plotter}; use crate::SavedSample; use criterion_plot::Size; use serde::Serialize; @@ -84,6 +84,7 @@ struct SummaryContext { violin_plot: Option, line_chart: Option, + line_throughput_chart: Option, benchmarks: Vec, } @@ -759,15 +760,32 @@ impl Html { let value_types: Vec<_> = data.iter().map(|&&(id, _)| id.value_type()).collect(); let mut line_path = None; + let mut line_throughput_path = None; if value_types.iter().all(|x| x == &value_types[0]) { if let Some(value_type) = value_types[0] { let values: Vec<_> = data.iter().map(|&&(id, _)| id.as_number()).collect(); if values.iter().any(|x| x != &values[0]) { - self.plotter - .borrow_mut() - .line_comparison(plot_ctx, formatter, data, value_type); + self.plotter.borrow_mut().line_comparison( + LinePlotConfig::time(), + plot_ctx, + formatter, + data, + value_type, + ); line_path = Some(plot_ctx.line_comparison_path()); + + // value_types being all equal implies throughput types being all equal + if data[0].0.throughput.is_some() { + self.plotter.borrow_mut().line_comparison( + LinePlotConfig::throughput(), + plot_ctx, + formatter, + data, + value_type, + ); + line_throughput_path = Some(plot_ctx.line_throughput_comparison_path()); + } } } } @@ -788,6 +806,7 @@ impl Html { violin_plot: Some(plot_ctx.violin_path().to_string_lossy().into_owned()), line_chart: line_path.map(|p| p.to_string_lossy().into_owned()), + line_throughput_chart: line_throughput_path.map(|p| p.to_string_lossy().into_owned()), benchmarks, }; diff --git a/src/html/summary_report.html.tt b/src/html/summary_report.html.tt index 4f36f62ba..e6f431060 100644 --- a/src/html/summary_report.html.tt +++ b/src/html/summary_report.html.tt @@ -66,6 +66,11 @@ Line Chart

This chart shows the mean measured time for each function as the input (or the size of the input) increases.

{{- endif }} + {{- if line_throughput_chart }} +

Throughput Chart

+ Line Chart +

This chart shows the mean measured throughput for each function as the input (or the size of the input) increases.

+ {{- endif }} {{- for bench in benchmarks }}
@@ -106,4 +111,4 @@ - \ No newline at end of file + diff --git a/src/lib.rs b/src/lib.rs index 855c68ff2..8419a0467 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1168,7 +1168,7 @@ https://bheisler.github.io/criterion.rs/book/faq.html /// // Now we can perform benchmarks with this group /// group.bench_function("Bench 1", |b| b.iter(|| 1 )); /// group.bench_function("Bench 2", |b| b.iter(|| 2 )); - /// + /// /// group.finish(); /// } /// criterion_group!(benches, bench_simple); diff --git a/src/plot/gnuplot_backend/mod.rs b/src/plot/gnuplot_backend/mod.rs index 27cc48be3..0e38878c9 100644 --- a/src/plot/gnuplot_backend/mod.rs +++ b/src/plot/gnuplot_backend/mod.rs @@ -22,7 +22,7 @@ use crate::measurement::ValueFormatter; use crate::report::{BenchmarkId, ValueType}; use crate::stats::bivariate::Data; -use super::{PlotContext, PlotData, Plotter}; +use super::{LinePlotConfig, PlotContext, PlotData, Plotter}; use crate::format; fn gnuplot_escape(string: &str) -> String { @@ -201,13 +201,15 @@ impl Plotter for Gnuplot { fn line_comparison( &mut self, + line_config: LinePlotConfig, ctx: PlotContext<'_>, formatter: &dyn ValueFormatter, all_curves: &[&(&BenchmarkId, Vec)], value_type: ValueType, ) { - let path = ctx.line_comparison_path(); + let path = (line_config.path)(&ctx); self.process_list.push(line_comparison( + line_config, formatter, ctx.id.as_title(), all_curves, diff --git a/src/plot/gnuplot_backend/summary.rs b/src/plot/gnuplot_backend/summary.rs index e5d2ab6be..34aa2d132 100644 --- a/src/plot/gnuplot_backend/summary.rs +++ b/src/plot/gnuplot_backend/summary.rs @@ -2,6 +2,7 @@ use super::{debug_script, gnuplot_escape}; use super::{DARK_BLUE, DEFAULT_FONT, KDE_POINTS, LINEWIDTH, POINT_SIZE, SIZE}; use crate::kde; use crate::measurement::ValueFormatter; +use crate::plot::LinePlotConfig; use crate::report::{BenchmarkId, ValueType}; use crate::stats::univariate::Sample; use crate::AxisScale; @@ -33,7 +34,8 @@ impl AxisScale { } #[cfg_attr(feature = "cargo-clippy", allow(clippy::explicit_counter_loop))] -pub fn line_comparison( +pub(crate) fn line_comparison( + line_cfg: LinePlotConfig, formatter: &dyn ValueFormatter, title: &str, all_curves: &[&(&BenchmarkId, Vec)], @@ -65,18 +67,22 @@ pub fn line_comparison( let mut i = 0; - let max = all_curves + let (max_id, max) = all_curves .iter() - .map(|&(_, data)| Sample::new(data).mean()) - .fold(::std::f64::NAN, f64::max); + .map(|&(id, data)| (*id, Sample::new(data).mean())) + .fold(None, |prev: Option<(&BenchmarkId, f64)>, next| match prev { + Some(prev) if prev.1 >= next.1 => Some(prev), + _ => Some(next), + }) + .unwrap(); - let mut dummy = [1.0]; - let unit = formatter.scale_values(max, &mut dummy); + let mut max_formatted = [max]; + let unit = (line_cfg.scale)(formatter, max_id, max, max_id, &mut max_formatted); f.configure(Axis::LeftY, |a| { a.configure(Grid::Major, |g| g.show()) .configure(Grid::Minor, |g| g.hide()) - .set(Label(format!("Average time ({})", unit))) + .set(Label(format!("Average {} ({})", line_cfg.label, unit))) .set(axis_scale.to_gnuplot()) }); @@ -89,14 +95,15 @@ pub fn line_comparison( // Unwrap is fine here because it will only fail if the assumptions above are not true // ie. programmer error. let x = id.as_number().unwrap(); - let y = Sample::new(sample).mean(); + let mut y = [Sample::new(sample).mean()]; + + (line_cfg.scale)(formatter, max_id, max, id, &mut y); - (x, y) + (x, y[0]) }) .collect(); tuples.sort_by(|&(ax, _), &(bx, _)| (ax.partial_cmp(&bx).unwrap_or(Ordering::Less))); - let (xs, mut ys): (Vec<_>, Vec<_>) = tuples.into_iter().unzip(); - formatter.scale_values(max, &mut ys); + let (xs, ys): (Vec<_>, Vec<_>) = tuples.into_iter().unzip(); let function_name = key.as_ref().map(|string| gnuplot_escape(string)); diff --git a/src/plot/mod.rs b/src/plot/mod.rs index 4bce39468..6d534876f 100644 --- a/src/plot/mod.rs +++ b/src/plot/mod.rs @@ -9,6 +9,7 @@ pub(crate) use plotters_backend::PlottersBackend; use crate::estimate::Statistic; use crate::measurement::ValueFormatter; use crate::report::{BenchmarkId, ComparisonData, MeasurementData, ReportContext, ValueType}; +use crate::Throughput; use std::path::PathBuf; const REPORT_STATS: [Statistic; 7] = [ @@ -50,6 +51,14 @@ impl<'a> PlotContext<'a> { path } + pub fn line_throughput_comparison_path(&self) -> PathBuf { + let mut path = self.context.output_directory.clone(); + path.push(self.id.as_directory_name()); + path.push("report"); + path.push("lines_throughput.svg"); + path + } + pub fn violin_path(&self) -> PathBuf { let mut path = self.context.output_directory.clone(); path.push(self.id.as_directory_name()); @@ -73,6 +82,53 @@ impl<'a> PlotData<'a> { } } +#[derive(Clone, Copy)] +pub(crate) struct LinePlotConfig { + label: &'static str, + scale: fn(&dyn ValueFormatter, &BenchmarkId, f64, &BenchmarkId, &mut [f64]) -> &'static str, + path: fn(&PlotContext<'_>) -> PathBuf, +} + +impl LinePlotConfig { + pub fn time() -> Self { + Self { + label: "time", + scale: |formatter, _, max, _, vals| formatter.scale_values(max, vals), + path: |ctx| ctx.line_comparison_path(), + } + } + + pub fn throughput() -> Self { + Self { + label: "throughput", + scale: |formatter, max_id, max, id, vals| { + // Scale values to be in line with max_id throughput + let from = id + .throughput + .as_ref() + .expect("Throughput chart expects throughput to be defined"); + let to = max_id.throughput.as_ref().unwrap(); + + let (from_bytes, to_bytes) = match (from, to) { + (Throughput::Bytes(from), Throughput::Bytes(to)) => (from, to), + (Throughput::BytesDecimal(from), Throughput::BytesDecimal(to)) => (from, to), + (Throughput::Elements(from), Throughput::Elements(to)) => (from, to), + _ => unreachable!("throughput types expected to be equal"), + }; + + let mul = *to_bytes as f64 / *from_bytes as f64; + + for val in vals.iter_mut() { + *val *= mul; + } + + formatter.scale_throughputs(max, to, vals) + }, + path: |ctx| ctx.line_throughput_comparison_path(), + } + } +} + pub(crate) trait Plotter { fn pdf(&mut self, ctx: PlotContext<'_>, data: PlotData<'_>); @@ -86,6 +142,7 @@ pub(crate) trait Plotter { fn line_comparison( &mut self, + line_config: LinePlotConfig, ctx: PlotContext<'_>, formatter: &dyn ValueFormatter, all_curves: &[&(&BenchmarkId, Vec)], diff --git a/src/plot/plotters_backend/mod.rs b/src/plot/plotters_backend/mod.rs index 4cd1b183d..12fc2f07d 100644 --- a/src/plot/plotters_backend/mod.rs +++ b/src/plot/plotters_backend/mod.rs @@ -1,4 +1,4 @@ -use super::{PlotContext, PlotData, Plotter}; +use super::{LinePlotConfig, PlotContext, PlotData, Plotter}; use crate::measurement::ValueFormatter; use crate::report::{BenchmarkId, ComparisonData, MeasurementData, ValueType}; use plotters::data::float::pretty_print_float; @@ -184,13 +184,15 @@ impl Plotter for PlottersBackend { fn line_comparison( &mut self, + line_config: LinePlotConfig, ctx: PlotContext<'_>, formatter: &dyn ValueFormatter, all_curves: &[&(&BenchmarkId, Vec)], value_type: ValueType, ) { - let path = ctx.line_comparison_path(); + let path = (line_config.path)(&ctx); summary::line_comparison( + line_config, formatter, ctx.id.as_title(), all_curves, diff --git a/src/plot/plotters_backend/summary.rs b/src/plot/plotters_backend/summary.rs index 0ebb851e2..c69f8627f 100644 --- a/src/plot/plotters_backend/summary.rs +++ b/src/plot/plotters_backend/summary.rs @@ -20,7 +20,8 @@ static COMPARISON_COLORS: [RGBColor; NUM_COLORS] = [ RGBColor(0, 255, 127), ]; -pub fn line_comparison( +pub(crate) fn line_comparison( + line_cfg: LinePlotConfig, formatter: &dyn ValueFormatter, title: &str, all_curves: &[&(&BenchmarkId, Vec)], @@ -28,7 +29,7 @@ pub fn line_comparison( value_type: ValueType, axis_scale: AxisScale, ) { - let (unit, series_data) = line_comparison_series_data(formatter, all_curves); + let (unit, series_data) = line_comparison_series_data(line_cfg, formatter, all_curves); let x_range = plotters::data::fitting_range(series_data.iter().flat_map(|(_, xs, _)| xs.iter())); @@ -40,10 +41,17 @@ pub fn line_comparison( .unwrap(); match axis_scale { - AxisScale::Linear => { - draw_line_comarision_figure(root_area, unit, x_range, y_range, value_type, series_data) - } + AxisScale::Linear => draw_line_comarision_figure( + line_cfg, + root_area, + unit, + x_range, + y_range, + value_type, + series_data, + ), AxisScale::Logarithmic => draw_line_comarision_figure( + line_cfg, root_area, unit, x_range.log_scale(), @@ -55,6 +63,7 @@ pub fn line_comparison( } fn draw_line_comarision_figure, YR: AsRangedCoord>( + line_cfg: LinePlotConfig, root_area: DrawingArea, y_unit: &str, x_range: XR, @@ -82,7 +91,7 @@ fn draw_line_comarision_figure, YR: AsRangedCoord .configure_mesh() .disable_mesh() .x_desc(format!("Input{}", input_suffix)) - .y_desc(format!("Average time ({})", y_unit)) + .y_desc(format!("Average {} ({})", line_cfg.label, y_unit)) .draw() .unwrap(); @@ -115,16 +124,21 @@ fn draw_line_comarision_figure, YR: AsRangedCoord #[allow(clippy::type_complexity)] fn line_comparison_series_data<'a>( + line_cfg: LinePlotConfig, formatter: &dyn ValueFormatter, all_curves: &[&(&'a BenchmarkId, Vec)], ) -> (&'static str, Vec<(Option<&'a String>, Vec, Vec)>) { - let max = all_curves + let (max_id, max) = all_curves .iter() - .map(|&(_, data)| Sample::new(data).mean()) - .fold(::std::f64::NAN, f64::max); + .map(|&(id, data)| (*id, Sample::new(data).mean())) + .fold(None, |prev: Option<(&BenchmarkId, f64)>, next| match prev { + Some(prev) if prev.1 >= next.1 => Some(prev), + _ => Some(next), + }) + .unwrap(); - let mut dummy = [1.0]; - let unit = formatter.scale_values(max, &mut dummy); + let mut max_formatted = [max]; + let unit = (line_cfg.scale)(formatter, max_id, max, max_id, &mut max_formatted); let mut series_data = vec![]; @@ -137,15 +151,16 @@ fn line_comparison_series_data<'a>( // Unwrap is fine here because it will only fail if the assumptions above are not true // ie. programmer error. let x = id.as_number().unwrap(); - let y = Sample::new(sample).mean(); + let mut y = [Sample::new(&sample).mean()]; + + (line_cfg.scale)(formatter, max_id, max, id, &mut y); - (x, y) + (x, y[0]) }) .collect(); tuples.sort_by(|&(ax, _), &(bx, _)| (ax.partial_cmp(&bx).unwrap_or(Ordering::Less))); let function_name = key.as_ref(); - let (xs, mut ys): (Vec<_>, Vec<_>) = tuples.into_iter().unzip(); - formatter.scale_values(max, &mut ys); + let (xs, ys): (Vec<_>, Vec<_>) = tuples.into_iter().unzip(); series_data.push((function_name, xs, ys)); } (unit, series_data)