From 029a8f2f3032e140ea0d0521aac81137988864cf Mon Sep 17 00:00:00 2001 From: Aidan De Angelis Date: Sat, 30 Nov 2024 08:44:34 -0800 Subject: [PATCH] feat(max-fail): introduce --max-fail runner option --- cargo-nextest/src/dispatch.rs | 33 +++++++++++++++++++++--- nextest-runner/src/reporter/displayer.rs | 5 ++-- nextest-runner/src/runner.rs | 24 ++++++++--------- site/src/docs/running.md | 3 +++ 4 files changed, 47 insertions(+), 18 deletions(-) diff --git a/cargo-nextest/src/dispatch.rs b/cargo-nextest/src/dispatch.rs index 4f3bfb348b7..8a9f9349080 100644 --- a/cargo-nextest/src/dispatch.rs +++ b/cargo-nextest/src/dispatch.rs @@ -845,9 +845,24 @@ pub struct TestRunnerOpts { fail_fast: bool, /// Run all tests regardless of failure - #[arg(long, conflicts_with = "no-run", overrides_with = "fail-fast")] + #[arg( + long, + name = "no-fail-fast", + conflicts_with = "no-run", + overrides_with = "fail-fast" + )] no_fail_fast: bool, + /// Number of tests that can fail before exiting test run + #[arg( + long, + name = "max-fail", + value_name = "N", + conflicts_with_all = &["no-run", "no-fail-fast"], + overrides_with = "fail-fast" + )] + max_fail: Option, + /// Behavior if there are no tests to run [default: fail] #[arg( long, @@ -887,10 +902,12 @@ impl TestRunnerOpts { if let Some(retries) = self.retries { builder.set_retries(RetryPolicy::new_without_delay(retries)); } - if self.no_fail_fast { - builder.set_fail_fast(false); + if let Some(max_fail) = self.max_fail { + builder.set_max_fail(max_fail); + } else if self.no_fail_fast { + // Ignore --fail-fast } else if self.fail_fast { - builder.set_fail_fast(true); + builder.set_max_fail(1); } if let Some(test_threads) = self.test_threads { builder.set_test_threads(test_threads); @@ -2420,6 +2437,7 @@ mod tests { "cargo nextest run --no-run --no-fail-fast", ArgumentConflict, ), + ("cargo nextest run --no-run --max-fail 3", ArgumentConflict), ( "cargo nextest run --no-run --failure-output immediate", ArgumentConflict, @@ -2437,6 +2455,13 @@ mod tests { ArgumentConflict, ), // --- + // --max-fail and these options conflict + // --- + ( + "cargo nextest run --max-fail 3 --no-fail-fast", + ArgumentConflict, + ), + // --- // Reuse build options conflict with cargo options // --- ( diff --git a/nextest-runner/src/reporter/displayer.rs b/nextest-runner/src/reporter/displayer.rs index 79b519211ae..526940737d1 100644 --- a/nextest-runner/src/reporter/displayer.rs +++ b/nextest-runner/src/reporter/displayer.rs @@ -1982,7 +1982,7 @@ fn write_final_warnings( if cancel_status == Some(CancelReason::TestFailure) { writeln!( writer, - "{}: {}/{} {} {} not run due to {} (run with {} to run all tests)", + "{}: {}/{} {} {} not run due to {} (run with {} to run all tests, or run with {})", "warning".style(styles.skip), not_run.style(styles.count), initial_run_count.style(styles.count), @@ -1990,6 +1990,7 @@ fn write_final_warnings( plural::were_plural_if(initial_run_count != 1 || not_run != 1), CancelReason::TestFailure.to_static_str().style(styles.skip), "--no-fail-fast".style(styles.count), + "--max-fail".style(styles.count), )?; } else { let due_to_reason = match cancel_status { @@ -2650,7 +2651,7 @@ mod tests { assert_eq!( warnings, "warning: 1/3 tests were not run due to test failure \ - (run with --no-fail-fast to run all tests)\n" + (run with --no-fail-fast to run all tests, or run with --max-fail)\n" ); let warnings = final_warnings_for( diff --git a/nextest-runner/src/runner.rs b/nextest-runner/src/runner.rs index a4baeb60c53..a0cafbf2956 100644 --- a/nextest-runner/src/runner.rs +++ b/nextest-runner/src/runner.rs @@ -132,7 +132,7 @@ impl Iterator for BackoffIter { pub struct TestRunnerBuilder { capture_strategy: CaptureStrategy, retries: Option, - fail_fast: Option, + max_fail: Option, test_threads: Option, } @@ -158,9 +158,9 @@ impl TestRunnerBuilder { self } - /// Sets the fail-fast value for this test runner. - pub fn set_fail_fast(&mut self, fail_fast: bool) -> &mut Self { - self.fail_fast = Some(fail_fast); + /// Sets the max-fail value for this test runner. + pub fn set_max_fail(&mut self, max_fail: usize) -> &mut Self { + self.max_fail = Some(max_fail); self } @@ -187,7 +187,7 @@ impl TestRunnerBuilder { .unwrap_or_else(|| profile.test_threads()) .compute(), }; - let fail_fast = self.fail_fast.unwrap_or_else(|| profile.fail_fast()); + let max_fail = self.max_fail.or_else(|| profile.fail_fast().then_some(1)); let runtime = Runtime::new().map_err(TestRunnerBuildError::TokioRuntimeCreate)?; let _guard = runtime.enter(); @@ -202,7 +202,7 @@ impl TestRunnerBuilder { cli_args, test_threads, force_retries: self.retries, - fail_fast, + max_fail, test_list, double_spawn, target_runner, @@ -308,7 +308,7 @@ struct TestRunnerInner<'a> { test_threads: usize, // This is Some if the user specifies a retry policy over the command-line. force_retries: Option, - fail_fast: bool, + max_fail: Option, test_list: &'a TestList<'a>, double_spawn: DoubleSpawnInfo, target_runner: TargetRunner, @@ -337,7 +337,7 @@ impl<'a> TestRunnerInner<'a> { self.profile.name(), self.cli_args.clone(), self.test_list.run_count(), - self.fail_fast, + self.max_fail, ); // Send the initial event. @@ -1869,7 +1869,7 @@ struct CallbackContext<'a, F> { cli_args: Vec, stopwatch: StopwatchStart, run_stats: RunStats, - fail_fast: bool, + max_fail: Option, running_setup_script: Option>, running_tests: BTreeMap, ContextTestInstance<'a>>, cancel_state: Option, @@ -1886,7 +1886,7 @@ where profile_name: &str, cli_args: Vec, initial_run_count: usize, - fail_fast: bool, + max_fail: Option, ) -> Self { Self { callback, @@ -1898,7 +1898,7 @@ where initial_run_count, ..RunStats::default() }, - fail_fast, + max_fail, running_setup_script: None, running_tests: BTreeMap::new(), cancel_state: None, @@ -2078,7 +2078,7 @@ where self.run_stats.on_test_finished(&run_statuses); // should this run be cancelled because of a failure? - let fail_cancel = self.fail_fast && !run_statuses.last_status().result.is_success(); + let fail_cancel = self.max_fail.map_or(false, |mf| self.run_stats.failed >= mf); self.callback(TestEventKind::TestFinished { test_instance, diff --git a/site/src/docs/running.md b/site/src/docs/running.md index bf16291a797..635475bde9f 100644 --- a/site/src/docs/running.md +++ b/site/src/docs/running.md @@ -187,6 +187,9 @@ cargo nextest run -E 'platform(host)' `--no-fail-fast` : Do not exit the test run on the first failure. Most useful for CI scenarios. +`--max-fail` +: Number of tests that can fail before aborting the test run. Useful for uncovering multiple issues without having to run the whole test suite. Mutually exclusive with `--no-fail-fast` + `-j`, `--test-threads` : Number of tests to run simultaneously. Note that this is separate from the number of build jobs to run simultaneously, which is specified by `--build-jobs`.