feat(max-fail): introduce --max-fail runner option

cargo-nextest/src/dispatch.rs

@@ -845,9 +845,24 @@ pub struct TestRunnerOpts {
     fail_fast: bool,
 
     /// Run all tests regardless of failure
-    #[arg(long, conflicts_with = "no-run", overrides_with = "fail-fast")]
+    #[arg(
+        long,
+        name = "no-fail-fast",
+        conflicts_with = "no-run",
+        overrides_with = "fail-fast"
+    )]
     no_fail_fast: bool,
 
+    /// Number of tests that can fail before exiting test run
+    #[arg(
+        long,
+        name = "max-fail",
+        value_name = "N",
+        conflicts_with_all = &["no-run", "no-fail-fast"],
+        overrides_with = "fail-fast"
+    )]
+    max_fail: Option<usize>,
+
     /// Behavior if there are no tests to run [default: fail]
     #[arg(
         long,
@@ -887,10 +902,12 @@ impl TestRunnerOpts {
         if let Some(retries) = self.retries {
             builder.set_retries(RetryPolicy::new_without_delay(retries));
         }
-        if self.no_fail_fast {
-            builder.set_fail_fast(false);
+        if let Some(max_fail) = self.max_fail {
+            builder.set_max_fail(max_fail);
+        } else if self.no_fail_fast {
+            // Ignore --fail-fast
         } else if self.fail_fast {
-            builder.set_fail_fast(true);
+            builder.set_max_fail(1);
         }
         if let Some(test_threads) = self.test_threads {
             builder.set_test_threads(test_threads);
@@ -2420,6 +2437,7 @@ mod tests {
                 "cargo nextest run --no-run --no-fail-fast",
                 ArgumentConflict,
             ),
+            ("cargo nextest run --no-run --max-fail 3", ArgumentConflict),
             (
                 "cargo nextest run --no-run --failure-output immediate",
                 ArgumentConflict,
@@ -2437,6 +2455,13 @@ mod tests {
                 ArgumentConflict,
             ),
             // ---
+            // --max-fail and these options conflict
+            // ---
+            (
+                "cargo nextest run --max-fail 3 --no-fail-fast",
+                ArgumentConflict,
+            ),
+            // ---
             // Reuse build options conflict with cargo options
             // ---
             (

nextest-runner/src/reporter/displayer.rs

@@ -1982,14 +1982,15 @@ fn write_final_warnings(
             if cancel_status == Some(CancelReason::TestFailure) {
                 writeln!(
                     writer,
-                    "{}: {}/{} {} {} not run due to {} (run with {} to run all tests)",
+                    "{}: {}/{} {} {} not run due to {} (run with {} to run all tests, or run with {})",
                     "warning".style(styles.skip),
                     not_run.style(styles.count),
                     initial_run_count.style(styles.count),
                     plural::tests_plural_if(initial_run_count != 1 || not_run != 1),
                     plural::were_plural_if(initial_run_count != 1 || not_run != 1),
                     CancelReason::TestFailure.to_static_str().style(styles.skip),
                     "--no-fail-fast".style(styles.count),
+                    "--max-fail".style(styles.count),
                 )?;
             } else {
                 let due_to_reason = match cancel_status {
@@ -2650,7 +2651,7 @@ mod tests {
         assert_eq!(
             warnings,
             "warning: 1/3 tests were not run due to test failure \
-             (run with --no-fail-fast to run all tests)\n"
+             (run with --no-fail-fast to run all tests, or run with --max-fail)\n"
         );
 
         let warnings = final_warnings_for(

nextest-runner/src/runner.rs

@@ -132,7 +132,7 @@ impl Iterator for BackoffIter {
 pub struct TestRunnerBuilder {
     capture_strategy: CaptureStrategy,
     retries: Option<RetryPolicy>,
-    fail_fast: Option<bool>,
+    max_fail: Option<usize>,
     test_threads: Option<TestThreads>,
 }
 
@@ -158,9 +158,9 @@ impl TestRunnerBuilder {
         self
     }
 
-    /// Sets the fail-fast value for this test runner.
-    pub fn set_fail_fast(&mut self, fail_fast: bool) -> &mut Self {
-        self.fail_fast = Some(fail_fast);
+    /// Sets the max-fail value for this test runner.
+    pub fn set_max_fail(&mut self, max_fail: usize) -> &mut Self {
+        self.max_fail = Some(max_fail);
         self
     }
 
@@ -187,7 +187,7 @@ impl TestRunnerBuilder {
                 .unwrap_or_else(|| profile.test_threads())
                 .compute(),
         };
-        let fail_fast = self.fail_fast.unwrap_or_else(|| profile.fail_fast());
+        let max_fail = self.max_fail.or_else(|| profile.fail_fast().then_some(1));
 
         let runtime = Runtime::new().map_err(TestRunnerBuildError::TokioRuntimeCreate)?;
         let _guard = runtime.enter();
@@ -202,7 +202,7 @@ impl TestRunnerBuilder {
                 cli_args,
                 test_threads,
                 force_retries: self.retries,
-                fail_fast,
+                max_fail,
                 test_list,
                 double_spawn,
                 target_runner,
@@ -308,7 +308,7 @@ struct TestRunnerInner<'a> {
     test_threads: usize,
     // This is Some if the user specifies a retry policy over the command-line.
     force_retries: Option<RetryPolicy>,
-    fail_fast: bool,
+    max_fail: Option<usize>,
     test_list: &'a TestList<'a>,
     double_spawn: DoubleSpawnInfo,
     target_runner: TargetRunner,
@@ -337,7 +337,7 @@ impl<'a> TestRunnerInner<'a> {
             self.profile.name(),
             self.cli_args.clone(),
             self.test_list.run_count(),
-            self.fail_fast,
+            self.max_fail,
         );
 
         // Send the initial event.
@@ -1869,7 +1869,7 @@ struct CallbackContext<'a, F> {
     cli_args: Vec<String>,
     stopwatch: StopwatchStart,
     run_stats: RunStats,
-    fail_fast: bool,
+    max_fail: Option<usize>,
     running_setup_script: Option<ContextSetupScript<'a>>,
     running_tests: BTreeMap<TestInstanceId<'a>, ContextTestInstance<'a>>,
     cancel_state: Option<CancelReason>,
@@ -1886,7 +1886,7 @@ where
         profile_name: &str,
         cli_args: Vec<String>,
         initial_run_count: usize,
-        fail_fast: bool,
+        max_fail: Option<usize>,
     ) -> Self {
         Self {
             callback,
@@ -1898,7 +1898,7 @@ where
                 initial_run_count,
                 ..RunStats::default()
             },
-            fail_fast,
+            max_fail,
             running_setup_script: None,
             running_tests: BTreeMap::new(),
             cancel_state: None,
@@ -2078,7 +2078,7 @@ where
                 self.run_stats.on_test_finished(&run_statuses);
 
                 // should this run be cancelled because of a failure?
-                let fail_cancel = self.fail_fast && !run_statuses.last_status().result.is_success();
+                let fail_cancel = self.max_fail.map_or(false, |mf| self.run_stats.failed >= mf);
 
                 self.callback(TestEventKind::TestFinished {
                     test_instance,

site/src/docs/running.md

@@ -187,6 +187,9 @@ cargo nextest run -E 'platform(host)'
 `--no-fail-fast`
 : Do not exit the test run on the first failure. Most useful for CI scenarios.
 
+`--max-fail`
+: Number of tests that can fail before aborting the test run. Useful for uncovering multiple issues without having to run the whole test suite. Mutually exclusive with `--no-fail-fast`
+
 `-j`, `--test-threads`
 : Number of tests to run simultaneously. Note that this is separate from the number of build jobs to run simultaneously, which is specified by `--build-jobs`.