From 5bbb5ca704fb0d4fd55ebb78cf6d62bfec405497 Mon Sep 17 00:00:00 2001 From: grandizzy Date: Mon, 10 Jun 2024 15:33:47 +0300 Subject: [PATCH] feat: add possibility to interrupt running test --- proptest/src/test_runner/errors.rs | 21 +++++++++ proptest/src/test_runner/replay.rs | 2 +- proptest/src/test_runner/runner.rs | 68 ++++++++++++++++++++++++++++++ 3 files changed, 90 insertions(+), 1 deletion(-) diff --git a/proptest/src/test_runner/errors.rs b/proptest/src/test_runner/errors.rs index bf17f201..1710c99a 100644 --- a/proptest/src/test_runner/errors.rs +++ b/proptest/src/test_runner/errors.rs @@ -25,6 +25,8 @@ use crate::test_runner::Reason; /// `Error::display()` into the `Fail` case. #[derive(Debug, Clone)] pub enum TestCaseError { + /// The test runner was interrupted. + Interrupt(Reason), /// The input was not valid for the test case. This does not count as a /// test failure (nor a success); rather, it simply signals to generate /// a new input and try again. @@ -61,6 +63,16 @@ pub type TestCaseResult = Result<(), TestCaseError>; pub(crate) type TestCaseResultV2 = Result; impl TestCaseError { + /// Interrupts this test case runner. This does not count as a test failure + /// (nor a success); rather, it simply signals to stop executing new runs, + /// regardless number of configured successful test cases. + /// + /// The string gives the location and context of the interruption, and + /// should be suitable for formatting like `Foo did X at {whence}`. + pub fn interrupt(reason: impl Into) -> Self { + TestCaseError::Interrupt(reason.into()) + } + /// Rejects the generated test input as invalid for this test case. This /// does not count as a test failure (nor a success); rather, it simply /// signals to generate a new input and try again. @@ -83,6 +95,9 @@ impl TestCaseError { impl fmt::Display for TestCaseError { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { match *self { + TestCaseError::Interrupt(ref why) => { + write!(f, "Case interrupted: {}", why) + } TestCaseError::Reject(ref whence) => { write!(f, "Input rejected at {}", whence) } @@ -101,6 +116,8 @@ impl From for TestCaseError { /// A failure state from running test cases for a single test. #[derive(Debug, Clone, PartialEq, Eq)] pub enum TestError { + /// The test was interrupted for the given reason. + Interrupt(Reason), /// The test was aborted for the given reason, for example, due to too many /// inputs having been rejected. Abort(Reason), @@ -113,6 +130,9 @@ pub enum TestError { impl fmt::Display for TestError { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { match *self { + TestError::Interrupt(ref why) => { + write!(f, "Test interrupted: {}", why) + } TestError::Abort(ref why) => write!(f, "Test aborted: {}", why), TestError::Fail(ref why, ref what) => { writeln!(f, "Test failed: {}.", why)?; @@ -127,6 +147,7 @@ impl fmt::Display for TestError { impl ::std::error::Error for TestError { fn description(&self) -> &str { match *self { + TestError::Interrupt(..) => "Interrupted", TestError::Abort(..) => "Abort", TestError::Fail(..) => "Fail", } diff --git a/proptest/src/test_runner/replay.rs b/proptest/src/test_runner/replay.rs index 4365d553..9b2fbf50 100644 --- a/proptest/src/test_runner/replay.rs +++ b/proptest/src/test_runner/replay.rs @@ -83,7 +83,7 @@ fn step_to_char(step: &TestCaseResult) -> char { match *step { Ok(_) => '+', Err(TestCaseError::Reject(_)) => '!', - Err(TestCaseError::Fail(_)) => '-', + Err(TestCaseError::Fail(_)) | Err(TestCaseError::Interrupt(_)) => '-', } } diff --git a/proptest/src/test_runner/runner.rs b/proptest/src/test_runner/runner.rs index d209dff6..e15e6b58 100644 --- a/proptest/src/test_runner/runner.rs +++ b/proptest/src/test_runner/runner.rs @@ -281,6 +281,14 @@ where match result { Ok(()) => verbose_message!(runner, TRACE, "Test case passed"), + Err(TestCaseError::Interrupt(ref reason)) => { + verbose_message!( + runner, + INFO_LOG, + "Test case interrupted: {}", + reason + ) + } Err(TestCaseError::Reject(ref reason)) => { verbose_message!(runner, INFO_LOG, "Test case rejected: {}", reason) } @@ -620,6 +628,12 @@ impl TestRunner { &mut fork_output, false, ); + + if let Err(TestError::Interrupt(_)) = result { + // exit runs loop if test was interrupted + break; + } + if let Err(TestError::Fail(_, ref value)) = result { if let Some(ref mut failure_persistence) = self.config.failure_persistence @@ -746,6 +760,9 @@ impl TestRunner { .unwrap_or(why); Err(TestError::Fail(why, case.current())) } + Err(TestCaseError::Interrupt(why)) => { + Err(TestError::Interrupt(why)) + } Err(TestCaseError::Reject(whence)) => { self.reject_global(whence)?; Ok(TestCaseOk::Reject) @@ -883,6 +900,7 @@ impl TestRunner { break; } } + Err(TestCaseError::Interrupt(_)) => {} } } } @@ -1094,6 +1112,56 @@ mod test { assert_eq!(Ok(()), result); } + #[test] + fn test_interrupt_at_run() { + let mut runner = TestRunner::new(Config { + failure_persistence: None, + cases: 20, + ..Config::default() + }); + + let run_count = RefCell::new(0); + let result = runner.run(&(1u32..), |_v| { + *run_count.borrow_mut() += 1; + if *run_count.borrow() == 10 { + return Err(TestCaseError::Interrupt(Reason::from( + "test interrupted", + ))); + } + Ok(()) + }); + // only 10 runs performed + assert_eq!(run_count.into_inner(), 10); + // interrupt does not count as fail + assert_eq!(Ok(()), result); + } + + #[test] + fn test_interrupt_with_failed_case() { + let mut runner = TestRunner::new(Config { + failure_persistence: None, + cases: 20, + ..Config::default() + }); + + let run_count = RefCell::new(0); + let _ = runner.run(&(0u32..10u32), |v| { + *run_count.borrow_mut() += 1; + if v < 5 { + if *run_count.borrow() == 10 { + return Err(TestCaseError::Interrupt(Reason::from( + "test interrupted", + ))); + } + Ok(()) + } else { + Err(TestCaseError::fail("not less than 5")) + } + }); + // no more than 10 runs + assert!(run_count.into_inner() <= 10); + } + #[test] fn test_fail_via_result() { let mut runner = TestRunner::new(Config {