[nextest-runner] add support for setup scripts (#977)

This is a long-requested feature! Currently landing this in an experimental state, with the aim being to gather feedback before stabilizing it. For more, see https://nexte.st/book/setup-scripts.

cargo-nextest/src/dispatch.rs

@@ -16,8 +16,9 @@ use nextest_metadata::{BinaryListSummary, BuildPlatform};
 use nextest_runner::{
     cargo_config::{CargoConfigs, EnvironmentMap, TargetTriple},
     config::{
-        get_num_cpus, NextestConfig, NextestProfile, NextestVersionConfig, NextestVersionEval,
-        PreBuildPlatform, RetryPolicy, TestGroup, TestThreads, ToolConfigFile, VersionOnlyConfig,
+        get_num_cpus, ConfigExperimental, NextestConfig, NextestProfile, NextestVersionConfig,
+        NextestVersionEval, PreBuildPlatform, RetryPolicy, TestGroup, TestThreads, ToolConfigFile,
+        VersionOnlyConfig,
     },
     double_spawn::DoubleSpawnInfo,
     errors::WriteTestListError,
@@ -29,7 +30,7 @@ use nextest_runner::{
     platform::BuildPlatforms,
     reporter::{FinalStatusLevel, StatusLevel, TestOutputDisplay, TestReporterBuilder},
     reuse_build::{archive_to_file, ArchiveReporter, MetadataOrPath, PathMapper, ReuseBuildInfo},
-    runner::{configure_handle_inheritance, TestRunnerBuilder},
+    runner::{configure_handle_inheritance, RunStatsFailureKind, TestRunnerBuilder},
     show_config::{ShowNextestVersion, ShowTestGroupSettings, ShowTestGroups, ShowTestGroupsMode},
     signal::SignalHandlerKind,
     target_runner::{PlatformRunner, TargetRunner},
@@ -39,6 +40,7 @@ use once_cell::sync::OnceCell;
 use owo_colors::{OwoColorize, Style};
 use semver::Version;
 use std::{
+    collections::BTreeSet,
     env::VarError,
     fmt::Write as _,
     io::{Cursor, Write},
@@ -229,12 +231,14 @@ impl ConfigOpts {
         &self,
         workspace_root: &Utf8Path,
         graph: &PackageGraph,
+        experimental: &BTreeSet<ConfigExperimental>,
     ) -> Result<NextestConfig> {
         NextestConfig::from_sources(
             workspace_root,
             graph,
             self.config_file.as_deref(),
             &self.tool_config_files,
+            experimental,
         )
         .map_err(ExpectedError::config_parse_error)
     }
@@ -483,10 +487,12 @@ struct TestBuildFilter {
 }
 
 impl TestBuildFilter {
+    #[allow(clippy::too_many_arguments)]
     fn compute_test_list<'g>(
         &self,
         ctx: &TestExecuteContext<'_>,
         graph: &'g PackageGraph,
+        workspace_root: Utf8PathBuf,
         binary_list: Arc<BinaryList>,
         test_filter_builder: TestFilterBuilder,
         env: EnvironmentMap,
@@ -511,6 +517,7 @@ impl TestBuildFilter {
             test_artifacts,
             rust_build_meta,
             &test_filter_builder,
+            workspace_root,
             env,
             // TODO: do we need to allow customizing this?
             get_num_cpus(),
@@ -976,9 +983,23 @@ impl BaseApp {
             .make_version_only_config(&self.workspace_root)?;
         self.check_version_config_initial(version_only_config.nextest_version())?;
 
-        let config = self
-            .config_opts
-            .make_config(&self.workspace_root, self.graph())?;
+        let experimental = version_only_config.experimental();
+        if !experimental.is_empty() {
+            log::info!(
+                "experimental features enabled: {}",
+                experimental
+                    .iter()
+                    .map(|x| x.to_string())
+                    .collect::<Vec<_>>()
+                    .join(", ")
+            );
+        }
+
+        let config = self.config_opts.make_config(
+            &self.workspace_root,
+            self.graph(),
+            version_only_config.experimental(),
+        )?;
 
         Ok((version_only_config, config))
     }
@@ -1271,6 +1292,7 @@ impl App {
         self.build_filter.compute_test_list(
             ctx,
             self.base.graph(),
+            self.base.workspace_root.clone(),
             binary_list,
             test_filter_builder,
             env,
@@ -1465,7 +1487,7 @@ impl App {
 
         let runner = runner_builder.build(
             &test_list,
-            profile,
+            &profile,
             handler,
             double_spawn.clone(),
             target_runner.clone(),
@@ -1479,7 +1501,20 @@ impl App {
         self.base
             .check_version_config_final(version_only_config.nextest_version())?;
         if !run_stats.is_success() {
-            return Err(ExpectedError::test_run_failed());
+            match run_stats.failure_kind() {
+                Some(RunStatsFailureKind::SetupScript) => {
+                    return Err(ExpectedError::setup_script_failed());
+                }
+                Some(RunStatsFailureKind::Test) => {
+                    return Err(ExpectedError::test_run_failed());
+                }
+                None => {
+                    // XXX This means that the final number run of tests was less than the initial
+                    // number. Why can this be except if tests were failed or canceled for some
+                    // reason?
+                    return Err(ExpectedError::test_run_failed());
+                }
+            }
         }
         Ok(())
     }

cargo-nextest/src/errors.rs

@@ -184,6 +184,8 @@ pub enum ExpectedError {
         #[from]
         err: ShowTestGroupsError,
     },
+    #[error("setup script failed")]
+    SetupScriptFailed,
     #[error("test run failed")]
     TestRunFailed,
     #[cfg(feature = "self-update")]
@@ -352,6 +354,10 @@ impl ExpectedError {
         Self::FilterExpressionParseError { all_errors }
     }
 
+    pub(crate) fn setup_script_failed() -> Self {
+        Self::SetupScriptFailed
+    }
+
     pub(crate) fn test_run_failed() -> Self {
         Self::TestRunFailed
     }
@@ -374,7 +380,6 @@ impl ExpectedError {
             | Self::StoreDirCreateError { .. }
             | Self::RootManifestNotFound { .. }
             | Self::CargoConfigError { .. }
-            | Self::ConfigParseError { .. }
             | Self::TestFilterBuilderError { .. }
             | Self::UnknownHostPlatform { .. }
             | Self::ArgumentFileReadError { .. }
@@ -390,6 +395,15 @@ impl ExpectedError {
             | Self::DialoguerError { .. }
             | Self::SignalHandlerSetupError { .. }
             | Self::ShowTestGroupsError { .. } => NextestExitCode::SETUP_ERROR,
+            Self::ConfigParseError { err } => {
+                // Experimental features not being enabled are their own error.
+                match err.kind() {
+                    ConfigParseErrorKind::ExperimentalFeatureNotEnabled { .. } => {
+                        NextestExitCode::EXPERIMENTAL_FEATURE_NOT_ENABLED
+                    }
+                    _ => NextestExitCode::SETUP_ERROR,
+                }
+            }
             Self::RequiredVersionNotMet { .. } => NextestExitCode::REQUIRED_VERSION_NOT_MET,
             #[cfg(feature = "self-update")]
             Self::UpdateVersionParseError { .. } => NextestExitCode::SETUP_ERROR,
@@ -402,6 +416,7 @@ impl ExpectedError {
             Self::BuildExecFailed { .. } | Self::BuildFailed { .. } => {
                 NextestExitCode::BUILD_FAILED
             }
+            Self::SetupScriptFailed => NextestExitCode::SETUP_SCRIPT_FAILED,
             Self::TestRunFailed => NextestExitCode::TEST_RUN_FAILED,
             Self::ArchiveCreateError { .. } => NextestExitCode::ARCHIVE_CREATION_FAILED,
             Self::WriteTestListError { .. } | Self::WriteEventError { .. } => {
@@ -497,7 +512,7 @@ impl ExpectedError {
             }
             Self::ConfigParseError { err } => {
                 match err.kind() {
-                    ConfigParseErrorKind::OverrideError(errors) => {
+                    ConfigParseErrorKind::CompiledDataParseError(errors) => {
                         // Override errors are printed out using miette.
                         for override_error in errors {
                             log::error!(
@@ -543,6 +558,57 @@ impl ExpectedError {
                         );
                         None
                     }
+                    ConfigParseErrorKind::UnknownConfigScripts {
+                        errors,
+                        known_scripts,
+                    } => {
+                        let known_scripts_str = known_scripts
+                            .iter()
+                            .map(|group_name| {
+                                group_name.if_supports_color_2(Stream::Stderr, |x| x.bold())
+                            })
+                            .join(", ");
+                        let mut errors_str = String::new();
+                        for error in errors {
+                            errors_str.push_str(&format!(
+                                " - script `{}` specified within profile `{}`\n",
+                                error.name.if_supports_color_2(Stream::Stderr, |x| x.bold()),
+                                error
+                                    .profile_name
+                                    .if_supports_color_2(Stream::Stderr, |x| x.bold())
+                            ));
+                        }
+
+                        log::error!(
+                            "for config file `{}`{}, unknown scripts defined \
+                        (known scripts: {known_scripts_str}):\n{errors_str}",
+                            err.config_file(),
+                            provided_by_tool(err.tool()),
+                        );
+                        None
+                    }
+                    ConfigParseErrorKind::UnknownExperimentalFeatures { unknown, known } => {
+                        let unknown_str = unknown
+                            .iter()
+                            .map(|feature_name| {
+                                feature_name.if_supports_color_2(Stream::Stderr, |x| x.bold())
+                            })
+                            .join(", ");
+                        let known_str = known
+                            .iter()
+                            .map(|feature_name| {
+                                feature_name.if_supports_color_2(Stream::Stderr, |x| x.bold())
+                            })
+                            .join(", ");
+
+                        log::error!(
+                            "for config file `{}`{}, unknown experimental features defined:
+                             {unknown_str} (known features: {known_str}):",
+                            err.config_file(),
+                            provided_by_tool(err.tool()),
+                        );
+                        None
+                    }
                     _ => {
                         // These other errors are printed out normally.
                         log::error!("{}", err);
@@ -677,6 +743,10 @@ impl ExpectedError {
                 log::error!("failed to write event to output");
                 Some(err as &dyn Error)
             }
+            Self::SetupScriptFailed => {
+                log::error!("setup script failed");
+                None
+            }
             Self::TestRunFailed => {
                 log::error!("test run failed");
                 None

fixtures/nextest-tests/.config/nextest.toml

@@ -1,11 +1,26 @@
 ## nextest config for this fixture
 
+# Note that these versions are not necessarily the version of nextest that's actually required --
+# they're only for testing with test_show_config_version.
 nextest-version = { required = "0.9.54", recommended = "0.9.56" }
 
+# This must be on one line for test_setup_scripts_not_enabled to work.
+experimental = ["setup-scripts"]
+
 [profile.default]
 # disable fail-fast to ensure a deterministic test run
 fail-fast = false
 
+[[profile.default.scripts]]
+platform = { host = "cfg(unix)" }
+filter = "test(=test_cargo_env_vars)"
+setup = "my-script-unix"
+
+[[profile.default.scripts]]
+platform = { host = "cfg(windows)" }
+filter = "test(=test_cargo_env_vars)"
+setup = "my-script-windows"
+
 [profile.with-retries]
 retries = 2
 
@@ -51,3 +66,9 @@ max-threads = 4
 
 [test-groups.unused]
 max-threads = 20
+
+[script.my-script-unix]
+command = './scripts/my-script.sh'
+
+[script.my-script-windows]
+command = 'cmd /c "scripts\\my-script.bat"'

fixtures/nextest-tests/scripts/my-script.bat

@@ -0,0 +1,4 @@
+REM If this environment variable is set, exit with non-zero.
+if defined __NEXTEST_SETUP_SCRIPT_ERROR exit 1
+
+ECHO MY_ENV_VAR=my-env-var>> %NEXTEST_ENV%

fixtures/nextest-tests/scripts/my-script.sh

@@ -0,0 +1,9 @@
+#!/bin/bash
+
+# If this environment variable is set, exit with non-zero.
+if [ ! -z "$__NEXTEST_SETUP_SCRIPT_ERROR" ]; then
+    echo "__NEXTEST_SETUP_SCRIPT_ERROR is set, exiting with 1"
+    exit 1
+fi
+
+echo MY_ENV_VAR=my-env-var >> $NEXTEST_ENV