Add libtest json output option (#1086)

Add support for machine-readable outputs for nextest's test runs.

cargo-nextest/src/dispatch.rs

@@ -28,7 +28,7 @@ use nextest_runner::{
     },
     partition::PartitionerBuilder,
     platform::BuildPlatforms,
-    reporter::{FinalStatusLevel, StatusLevel, TestOutputDisplay, TestReporterBuilder},
+    reporter::{structured, FinalStatusLevel, StatusLevel, TestOutputDisplay, TestReporterBuilder},
     reuse_build::{archive_to_file, ArchiveReporter, MetadataOrPath, PathMapper, ReuseBuildInfo},
     runner::{configure_handle_inheritance, RunStatsFailureKind, TestRunnerBuilder},
     show_config::{ShowNextestVersion, ShowTestGroupSettings, ShowTestGroups, ShowTestGroupsMode},
@@ -771,6 +771,18 @@ enum IgnoreOverridesOpt {
     All,
 }
 
+#[derive(Clone, Copy, Debug, ValueEnum, Default)]
+enum MessageFormat {
+    /// The default output format
+    #[default]
+    Human,
+    /// Output test information in the same format as libtest itself
+    LibtestJson,
+    /// Output test information in the same format as libtest itself, with an
+    /// `nextest` subobject that includes additional metadata
+    LibtestJsonPlus,
+}
+
 #[derive(Debug, Default, Args)]
 #[command(next_help_heading = "Reporter options")]
 struct TestReporterOpts {
@@ -818,6 +830,32 @@ struct TestReporterOpts {
     /// Do not display the progress bar
     #[arg(long, env = "NEXTEST_HIDE_PROGRESS_BAR")]
     hide_progress_bar: bool,
+
+    /// The format to use for outputting test results
+    #[arg(
+        long,
+        name = "message-format",
+        value_enum,
+        default_value_t,
+        conflicts_with = "no-run",
+        value_name = "FORMAT",
+        env = "NEXTEST_MESSAGE_FORMAT"
+    )]
+    message_format: MessageFormat,
+
+    /// The specific version of the `message-format` to use
+    ///
+    /// This version string allows the machine-readable formats to use a stable
+    /// structure for consistent consumption across changes to nextest. If not
+    /// specified the latest version for the selected message format is used
+    #[arg(
+        long,
+        conflicts_with = "no-run",
+        requires = "message-format",
+        value_name = "VERSION",
+        env = "NEXTEST_MESSAGE_FORMAT_VERSION"
+    )]
+    message_format_version: Option<String>,
 }
 
 impl TestReporterOpts {
@@ -1504,10 +1542,24 @@ impl App {
         let output = output_writer.reporter_output();
         let profile = profile.apply_build_platforms(&build_platforms);
 
+        let structured_reporter = match reporter_opts.message_format {
+            MessageFormat::Human => structured::StructuredReporter::Disabled,
+            MessageFormat::LibtestJson | MessageFormat::LibtestJsonPlus => {
+                structured::StructuredReporter::Libtest(structured::LibtestReporter::new(
+                    reporter_opts.message_format_version.as_deref(),
+                    if matches!(reporter_opts.message_format, MessageFormat::LibtestJsonPlus) {
+                        structured::EmitNextestObject::Yes
+                    } else {
+                        structured::EmitNextestObject::No
+                    },
+                )?)
+            }
+        };
+
         let mut reporter = reporter_opts
             .to_builder(no_capture)
             .set_verbose(self.base.output.verbose)
-            .build(&test_list, &profile, output);
+            .build(&test_list, &profile, output, structured_reporter);
         if self
             .base
             .output

cargo-nextest/src/errors.rs

@@ -242,6 +242,11 @@ pub enum ExpectedError {
         #[source]
         err: std::io::Error,
     },
+    #[error("message format version is not valid")]
+    InvalidMessageFormatVersion {
+        #[from]
+        err: FormatVersionError,
+    },
 }
 
 impl ExpectedError {
@@ -394,7 +399,8 @@ impl ExpectedError {
             | Self::TestBinaryArgsParseError { .. }
             | Self::DialoguerError { .. }
             | Self::SignalHandlerSetupError { .. }
-            | Self::ShowTestGroupsError { .. } => NextestExitCode::SETUP_ERROR,
+            | Self::ShowTestGroupsError { .. }
+            | Self::InvalidMessageFormatVersion { .. } => NextestExitCode::SETUP_ERROR,
             Self::ConfigParseError { err } => {
                 // Experimental features not being enabled are their own error.
                 match err.kind() {
@@ -835,6 +841,10 @@ impl ExpectedError {
                 log::error!("[double-spawn] failed to exec `{command:?}`");
                 Some(err as &dyn Error)
             }
+            Self::InvalidMessageFormatVersion { err } => {
+                log::error!("error parsing message format version`");
+                Some(err as &dyn Error)
+            }
         };
 
         while let Some(err) = next_error {

nextest-runner/src/errors.rs

@@ -1411,5 +1411,6 @@ mod self_update_errors {
     }
 }
 
+pub use crate::reporter::structured::FormatVersionError;
 #[cfg(feature = "self-update")]
 pub use self_update_errors::*;

nextest-runner/src/reporter.rs

@@ -6,6 +6,8 @@
 //! The main structure in this module is [`TestReporter`].
 
 mod aggregator;
+pub mod structured;
+
 use crate::{
     config::{NextestProfile, ScriptId},
     errors::WriteEventError,
@@ -161,6 +163,7 @@ pub struct TestReporterBuilder {
     success_output: Option<TestOutputDisplay>,
     status_level: Option<StatusLevel>,
     final_status_level: Option<FinalStatusLevel>,
+
     verbose: bool,
     hide_progress_bar: bool,
 }
@@ -220,6 +223,7 @@ impl TestReporterBuilder {
         test_list: &TestList,
         profile: &NextestProfile<'a>,
         output: ReporterStderr<'a>,
+        structured_reporter: structured::StructuredReporter<'a>,
     ) -> TestReporter<'a> {
         let styles = Box::default();
         let binary_id_width = test_list
@@ -319,6 +323,7 @@ impl TestReporterBuilder {
                 final_outputs: DebugIgnore(vec![]),
             },
             stderr,
+            structured_reporter,
             metadata_reporter: aggregator,
         }
     }
@@ -330,11 +335,15 @@ enum ReporterStderrImpl<'a> {
     Buffer(&'a mut Vec<u8>),
 }
 
-/// Functionality to report test results to stderr and JUnit
+/// Functionality to report test results to stderr, JUnit, and/or structured,
+/// machine-readable results to stdout
 pub struct TestReporter<'a> {
     inner: TestReporterImpl<'a>,
     stderr: ReporterStderrImpl<'a>,
+    /// Used to aggregate events for JUnit reports written to disk
     metadata_reporter: EventAggregator<'a>,
+    /// Used to emit test events in machine-readable format(s) to stdout
+    structured_reporter: structured::StructuredReporter<'a>,
 }
 
 impl<'a> TestReporter<'a> {
@@ -383,6 +392,8 @@ impl<'a> TestReporter<'a> {
                     .map_err(WriteEventError::Io)?;
             }
         }
+
+        self.structured_reporter.write_event(&event)?;
         self.metadata_reporter.write_event(event)?;
         Ok(())
     }
@@ -1780,7 +1791,9 @@ impl Styles {
 #[cfg(test)]
 mod tests {
     use super::*;
-    use crate::{config::NextestConfig, platform::BuildPlatforms};
+    use crate::{
+        config::NextestConfig, platform::BuildPlatforms, reporter::structured::StructuredReporter,
+    };
 
     #[test]
     fn no_capture_settings() {
@@ -1802,6 +1815,7 @@ mod tests {
             &test_list,
             &profile.apply_build_platforms(&build_platforms),
             output,
+            StructuredReporter::Disabled,
         );
         assert!(reporter.inner.no_capture, "no_capture is true");
         assert_eq!(

nextest-runner/src/reporter/structured.rs

@@ -0,0 +1,70 @@
+//! Functionality for emitting structured, machine readable output in different
+//! formats
+
+mod libtest;
+
+use super::*;
+pub use libtest::{EmitNextestObject, LibtestReporter};
+
+/// Error returned when a user-supplied format version fails to be parsed to a
+/// valid and supported version
+#[derive(Clone, Debug, thiserror::Error)]
+#[error("invalid format version: {input}")]
+pub struct FormatVersionError {
+    /// The input that failed to parse.
+    pub input: String,
+    /// The underlying error
+    #[source]
+    pub err: FormatVersionErrorInner,
+}
+
+/// The different errors that can occur when parsing and validating a format version
+#[derive(Clone, Debug, thiserror::Error)]
+pub enum FormatVersionErrorInner {
+    /// The input did not have a valid syntax
+    #[error("expected format version in form of `{expected}`")]
+    InvalidFormat {
+        /// The expected pseudo format
+        expected: &'static str,
+    },
+    /// A decimal integer was expected but could not be parsed
+    #[error("version component `{which}` could not be parsed as an integer")]
+    InvalidInteger {
+        /// Which component was invalid
+        which: &'static str,
+        /// The parse failure
+        #[source]
+        err: std::num::ParseIntError,
+    },
+    /// The version component was not within th expected range
+    #[error("version component `{which}` value {value} is out of range {range:?}")]
+    InvalidValue {
+        /// The component which was out of range
+        which: &'static str,
+        /// The value that was parsed
+        value: u8,
+        /// The range of valid values for the component
+        range: std::ops::Range<u8>,
+    },
+}
+
+/// A reporter for structured, machine readable, output based on the user's
+/// preference
+pub enum StructuredReporter<'a> {
+    /// Libtest compatible output
+    Libtest(LibtestReporter<'a>),
+    // TODO: make a custom format that is easier to consume than libtest's
+    //Json(json::JsonReporter<'a>),
+    /// Variant that doesn't actually emit anything
+    Disabled,
+}
+
+impl<'a> StructuredReporter<'a> {
+    #[inline]
+    pub(super) fn write_event(&mut self, event: &TestEvent<'a>) -> Result<(), WriteEventError> {
+        match self {
+            Self::Disabled => Ok(()),
+            Self::Libtest(ltr) => ltr.write_event(event),
+        }
+    }
+}