Skip to content

Commit

Permalink
[proptest] Minimal implementation of interceptor for excessive panics…
Browse files Browse the repository at this point in the history
… outputs
  • Loading branch information
target-san authored and matthew-russo committed Sep 22, 2024
1 parent 52d3a38 commit 23ef010
Show file tree
Hide file tree
Showing 13 changed files with 398 additions and 56 deletions.
3 changes: 3 additions & 0 deletions proptest/CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,9 @@
### New Features

- When running persisted regressions, the most recently added regression is now run first.
- Added `handle-panics` feature which enables catching panics raised in tests and turning them into failures
- Added `backtrace` feature which enables capturing backtraces for both test failures and panics,
if `handle-panics` feature is enabled

## 1.5.0

Expand Down
9 changes: 9 additions & 0 deletions proptest/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -56,6 +56,15 @@ atomic64bit = []

bit-set = [ "dep:bit-set", "dep:bit-vec" ]

# Enables proper handling of panics
# In particular, hides all intermediate panics flowing into stderr during shrink phase
handle-panics = ["std"]

# Enables gathering of failure backtraces
# * when test failure is reported via `prop_assert_*` macro
# * when normal assertion fails or panic fires, if `handle-panics` feature is enabled too
backtrace = ["std"]

[dependencies]
bitflags = "2"
unarray = "0.1.4"
Expand Down
2 changes: 1 addition & 1 deletion proptest/src/collection.rs
Original file line number Diff line number Diff line change
Expand Up @@ -705,7 +705,7 @@ mod test {

match result {
Ok(true) => num_successes += 1,
Err(TestError::Fail(_, value)) => {
Err(TestError::Fail(_, _, value)) => {
// The minimal case always has between 5 (due to min
// length) and 9 (min element value = 1) elements, and
// always sums to exactly 9.
Expand Down
2 changes: 1 addition & 1 deletion proptest/src/strategy/flatten.rs
Original file line number Diff line number Diff line change
Expand Up @@ -293,7 +293,7 @@ mod test {

match result {
Ok(_) => {}
Err(TestError::Fail(_, v)) => {
Err(TestError::Fail(_, _, v)) => {
failures += 1;
assert_eq!((10001, 10002), v);
}
Expand Down
8 changes: 4 additions & 4 deletions proptest/src/strategy/unions.rs
Original file line number Diff line number Diff line change
Expand Up @@ -495,8 +495,8 @@ mod test {

match result {
Ok(true) => passed += 1,
Err(TestError::Fail(_, 15)) => converged_low += 1,
Err(TestError::Fail(_, 30)) => converged_high += 1,
Err(TestError::Fail(_, _, 15)) => converged_low += 1,
Err(TestError::Fail(_, _, 30)) => converged_high += 1,
e => panic!("Unexpected result: {:?}", e),
}
}
Expand Down Expand Up @@ -572,8 +572,8 @@ mod test {

match result {
Ok(true) => passed += 1,
Err(TestError::Fail(_, 15)) => converged_low += 1,
Err(TestError::Fail(_, 30)) => converged_high += 1,
Err(TestError::Fail(_, _, 15)) => converged_low += 1,
Err(TestError::Fail(_, _, 30)) => converged_high += 1,
e => panic!("Unexpected result: {:?}", e),
}
}
Expand Down
22 changes: 13 additions & 9 deletions proptest/src/string.rs
Original file line number Diff line number Diff line change
Expand Up @@ -171,10 +171,7 @@ pub fn string_regex_parsed(expr: &Hir) -> ParseResult<String> {
/// [`regex` crate's documentation](https://docs.rs/regex/*/regex/#opt-out-of-unicode-support)
/// for more information.
pub fn bytes_regex(regex: &str) -> ParseResult<Vec<u8>> {
let hir = ParserBuilder::new()
.utf8(false)
.build()
.parse(regex)?;
let hir = ParserBuilder::new().utf8(false).build().parse(regex)?;
bytes_regex_parsed(&hir)
}

Expand Down Expand Up @@ -361,8 +358,8 @@ fn unsupported<T>(error: &'static str) -> Result<T, Error> {
mod test {
use std::collections::HashSet;

use regex::Regex;
use regex::bytes::Regex as BytesRegex;
use regex::Regex;

use super::*;

Expand Down Expand Up @@ -402,7 +399,8 @@ mod test {
max_distinct: usize,
iterations: usize,
) {
let generated = generate_byte_values_matching_regex(pattern, iterations);
let generated =
generate_byte_values_matching_regex(pattern, iterations);
assert!(
generated.len() >= min_distinct,
"Expected to generate at least {} strings, but only \
Expand Down Expand Up @@ -477,7 +475,8 @@ mod test {
if !ok {
panic!(
"Generated string {:?} which does not match {:?}",
printable_ascii(&s), pattern
printable_ascii(&s),
pattern
);
}

Expand Down Expand Up @@ -584,10 +583,15 @@ mod test {
fn test_non_utf8_byte_strings() {
do_test_bytes(r"(?-u)[\xC0-\xFF]\x20", 64, 64, 512);
do_test_bytes(r"(?-u)\x20[\x80-\xBF]", 64, 64, 512);
do_test_bytes(r#"(?x-u)
do_test_bytes(
r#"(?x-u)
\xed (( ( \xa0\x80 | \xad\xbf | \xae\x80 | \xaf\xbf )
( \xed ( \xb0\x80 | \xbf\xbf ) )? )
| \xb0\x80 | \xbe\x80 | \xbf\xbf )"#, 15, 15, 120);
| \xb0\x80 | \xbe\x80 | \xbf\xbf )"#,
15,
15,
120,
);
}

fn assert_send_and_sync<T: Send + Sync>(_: T) {}
Expand Down
5 changes: 4 additions & 1 deletion proptest/src/sugar.rs
Original file line number Diff line number Diff line change
Expand Up @@ -754,7 +754,10 @@ macro_rules! prop_assert {
let message = format!($($fmt)*);
let message = format!("{} at {}:{}", message, file!(), line!());
return ::core::result::Result::Err(
$crate::test_runner::TestCaseError::fail(message));
$crate::test_runner::TestCaseError::Fail(
message.into(),
$crate::test_runner::Backtrace::capture(),
));
}
};
}
Expand Down
135 changes: 135 additions & 0 deletions proptest/src/test_runner/backtrace.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,135 @@
//-
// Copyright 2024
//
// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
// option. This file may not be copied, modified, or distributed
// except according to those terms.

use core::fmt;
/// Holds test failure backtrace, if captured
///
/// If feature `backtrace` is disabled, it's a zero-sized struct with no logic
///
/// If `backtrace` is enabled, attempts to capture backtrace using `std::backtrace::Backtrace` -
/// if requested
#[derive(Clone, Default)]
pub struct Backtrace(internal::Backtrace);

impl Backtrace {
/// Creates empty backtrace object
///
/// Used when client code doesn't care
pub fn empty() -> Self {
Self(internal::Backtrace::empty())
}
/// Tells whether there's backtrace captured
pub fn is_empty(&self) -> bool {
self.0.is_empty()
}
/// Attempts to capture backtrace - but only if `backtrace` feature is enabled
#[inline(always)]
pub fn capture() -> Self {
Self(internal::Backtrace::capture())
}
}

impl fmt::Debug for Backtrace {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
fmt::Debug::fmt(&self.0, f)
}
}

impl fmt::Display for Backtrace {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
fmt::Display::fmt(&self.0, f)
}
}

#[cfg(feature = "backtrace")]
mod internal {
use core::fmt;
use std::backtrace as bt;
use std::sync::Arc;

// `std::backtrace::Backtrace` isn't `Clone`, so we have
// to use `Arc` to also maintain `Send + Sync`
#[derive(Clone, Default)]
pub struct Backtrace(Option<Arc<bt::Backtrace>>);

impl Backtrace {
pub fn empty() -> Self {
Self(None)
}

pub fn is_empty(&self) -> bool {
self.0.is_none()
}

#[inline(always)]
pub fn capture() -> Self {
let bt = bt::Backtrace::capture();
// Store only if we have backtrace
if bt.status() == bt::BacktraceStatus::Captured {
Self(Some(Arc::new(bt)))
} else {
Self(None)
}
}
}

impl fmt::Debug for Backtrace {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
if let Some(ref arc) = self.0 {
fmt::Debug::fmt(arc.as_ref(), f)
} else {
Ok(())
}
}
}

impl fmt::Display for Backtrace {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
if let Some(ref arc) = self.0 {
fmt::Display::fmt(arc.as_ref(), f)
} else {
Ok(())
}
}
}
}

#[cfg(not(feature = "backtrace"))]
mod internal {
use core::fmt;

#[derive(Clone, Default)]
pub struct Backtrace;

impl Backtrace {
pub fn empty() -> Self {
Self
}

pub fn is_empty(&self) -> bool {
true
}

pub fn capture() -> Self {
Self
}
}

impl fmt::Debug for Backtrace {
fn fmt(&self, _: &mut fmt::Formatter<'_>) -> fmt::Result {
Ok(())
}
}

impl fmt::Display for Backtrace {
fn fmt(&self, _: &mut fmt::Formatter<'_>) -> fmt::Result {
Ok(())
}
}
}
49 changes: 41 additions & 8 deletions proptest/src/test_runner/errors.rs
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
//-
// Copyright 2017, 2018 The proptest developers
// Copyright 2017, 2018, 2024 The proptest developers
//
// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
Expand All @@ -14,6 +14,8 @@ use std::string::ToString;

use crate::test_runner::Reason;

use super::Backtrace;

/// Errors which can be returned from test cases to indicate non-successful
/// completion.
///
Expand All @@ -30,7 +32,7 @@ pub enum TestCaseError {
/// a new input and try again.
Reject(Reason),
/// The code under test failed the test.
Fail(Reason),
Fail(Reason, Backtrace),
}

/// Indicates the type of test that ran successfully.
Expand Down Expand Up @@ -76,7 +78,7 @@ impl TestCaseError {
/// The string should indicate the location of the failure, but may
/// generally be any string.
pub fn fail(reason: impl Into<Reason>) -> Self {
TestCaseError::Fail(reason.into())
TestCaseError::Fail(reason.into(), Backtrace::empty())
}
}

Expand All @@ -86,7 +88,13 @@ impl fmt::Display for TestCaseError {
TestCaseError::Reject(ref whence) => {
write!(f, "Input rejected at {}", whence)
}
TestCaseError::Fail(ref why) => write!(f, "Case failed: {}", why),
TestCaseError::Fail(ref why, ref bt) => {
if bt.is_empty() {
write!(f, "Case failed: {why}")
} else {
write!(f, "Case failed: {why}\n{bt}")
}
}
}
}
}
Expand All @@ -99,24 +107,49 @@ impl<E: ::std::error::Error> From<E> for TestCaseError {
}

/// A failure state from running test cases for a single test.
#[derive(Debug, Clone, PartialEq, Eq)]
#[derive(Debug, Clone)]
pub enum TestError<T> {
/// The test was aborted for the given reason, for example, due to too many
/// inputs having been rejected.
Abort(Reason),
/// A failing test case was found. The string indicates where and/or why
/// the test failed. The `T` is the minimal input found to reproduce the
/// failure.
Fail(Reason, T),
Fail(Reason, Backtrace, T),
}

impl<T: PartialEq> PartialEq for TestError<T> {
fn eq(&self, other: &Self) -> bool {
match (self, other) {
(Self::Abort(l0), Self::Abort(r0)) => l0 == r0,
(Self::Fail(l0, _, l2), Self::Fail(r0, _, r2)) => {
l0 == r0 && l2 == r2
}
(TestError::Abort(_), TestError::Fail(_, _, _))
| (TestError::Fail(_, _, _), TestError::Abort(_)) => false,
}
}
}

impl<T: Eq> Eq for TestError<T> {}

impl<T: fmt::Debug> fmt::Display for TestError<T> {
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
match *self {
TestError::Abort(ref why) => write!(f, "Test aborted: {}", why),
TestError::Fail(ref why, ref what) => {
TestError::Fail(ref why, ref bt, ref what) => {
writeln!(f, "Test failed: {}.", why)?;
write!(f, "minimal failing input: {:#?}", what)

if !bt.is_empty() {
writeln!(f, "\nstack backtrace:\n{bt}")?;
// No need for extra newline, backtrace seems to print it anyway
} else {
// Extra empty line between failure description and minimal failing input
writeln!(f)?;
}

writeln!(f, "minimal failing input: {:#?}", what)?;
Ok(())
}
}
}
Expand Down
4 changes: 3 additions & 1 deletion proptest/src/test_runner/mod.rs
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
//-
// Copyright 2017, 2018 The proptest developers
// Copyright 2017, 2018, 2024 The proptest developers
//
// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
Expand All @@ -12,6 +12,7 @@
//! You do not normally need to access things in this module directly except
//! when implementing new low-level strategies.
mod backtrace;
mod config;
mod errors;
mod failure_persistence;
Expand All @@ -22,6 +23,7 @@ mod result_cache;
mod rng;
mod runner;

pub use self::backtrace::*;
pub use self::config::*;
pub use self::errors::*;
pub use self::failure_persistence::*;
Expand Down
Loading

0 comments on commit 23ef010

Please sign in to comment.