From 74270082f2b24a66a371a2e147cc50480ecbcc16 Mon Sep 17 00:00:00 2001 From: Rain Date: Mon, 9 Dec 2024 01:06:00 +0000 Subject: [PATCH] [nextest-runner] use a setup script to seed build for integration tests This is a really nice performance improvement on my local machine (goes from ~5s to ~3s) -- should see a similar or even better result in CI. --- .config/nextest.toml | 4 +- Cargo.lock | 40 ++++ Cargo.toml | 3 + cargo-nextest/src/dispatch.rs | 8 + cargo-nextest/src/errors.rs | 10 + fixture-data/src/nextest_tests.rs | 3 +- fixtures/nextest-tests/.config/nextest.toml | 3 + .../cdylib/cdylib-link/src/lib.rs | 1 - .../nextest-tests/proc-macro-test/src/lib.rs | 1 + fixtures/nextest-tests/tests/basic.rs | 32 ++-- integration-tests/Cargo.toml | 18 +- integration-tests/src/env.rs | 27 +++ integration-tests/src/lib.rs | 6 + integration-tests/src/nextest_cli.rs | 177 ++++++++++++++++++ integration-tests/src/seed.rs | 127 +++++++++++++ integration-tests/test-helpers/build-seed.rs | 40 ++++ .../tests/integration/fixtures.rs | 158 +--------------- integration-tests/tests/integration/main.rs | 21 ++- .../integration__archive_includes.snap | 5 +- ...gration__archive_includes_without_uds.snap | 4 +- ...integration__archive_missing_includes.snap | 3 +- .../integration__archive_no_includes.snap | 5 +- .../tests/integration/temp_project.rs | 69 ++++--- 23 files changed, 549 insertions(+), 216 deletions(-) create mode 100644 integration-tests/src/env.rs create mode 100644 integration-tests/src/lib.rs create mode 100644 integration-tests/src/nextest_cli.rs create mode 100644 integration-tests/src/seed.rs create mode 100644 integration-tests/test-helpers/build-seed.rs diff --git a/.config/nextest.toml b/.config/nextest.toml index 281b696168a..d5bb2148676 100644 --- a/.config/nextest.toml +++ b/.config/nextest.toml @@ -58,6 +58,4 @@ max-threads = 8 max-threads = 8 [script.cargo-fetch-fixture] -command = "cargo fetch --manifest-path fixtures/nextest-tests/Cargo.toml" -capture-stdout = true -capture-stderr = true +command = "cargo run --bin build-seed" diff --git a/Cargo.lock b/Cargo.lock index 9aa5bd31451..824aeb4fd97 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -629,6 +629,15 @@ version = "0.8.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" +[[package]] +name = "cp_r" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "837ca07dfd27a2663ac7c4701bb35856b534c2a61dd47af06ccf65d3bec79ebc" +dependencies = [ + "filetime", +] + [[package]] name = "cpufeatures" version = "0.2.14" @@ -940,6 +949,15 @@ dependencies = [ "percent-encoding", ] +[[package]] +name = "fs-err" +version = "3.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8bb60e7409f34ef959985bc9d9c5ee8f5db24ee46ed9775850548021710f807f" +dependencies = [ + "autocfg", +] + [[package]] name = "future-queue" version = "0.3.0" @@ -1439,8 +1457,11 @@ dependencies = [ "cfg-if", "clap", "color-eyre", + "cp_r", "enable-ansi-support", "fixture-data", + "fs-err", + "hex", "insta", "itertools", "nextest-metadata", @@ -1448,7 +1469,9 @@ dependencies = [ "pathdiff", "regex", "serde_json", + "sha2", "target-spec", + "whoami", ] [[package]] @@ -3531,6 +3554,12 @@ version = "0.11.0+wasi-snapshot-preview1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" +[[package]] +name = "wasite" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8dad83b4f25e74f184f64c43b150b91efe7647395b42289f38e50566d82855b" + [[package]] name = "wasm-bindgen" version = "0.2.95" @@ -3627,6 +3656,17 @@ dependencies = [ "rustls-pki-types", ] +[[package]] +name = "whoami" +version = "1.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "372d5b87f58ec45c384ba03563b03544dc5fadc3983e434b286913f5b4a9bb6d" +dependencies = [ + "redox_syscall", + "wasite", + "web-sys", +] + [[package]] name = "win32job" version = "2.0.0" diff --git a/Cargo.toml b/Cargo.toml index 0231da10a79..8e93a08aea2 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -45,6 +45,7 @@ config = { version = "0.14.1", default-features = false, features = [ chrono = "0.4.38" clap = { version = "4.5.23", features = ["derive"] } console-subscriber = "0.4.1" +cp_r = "0.5.2" crossterm = { version = "0.28.1", features = ["event-stream"] } dialoguer = "0.11.0" debug-ignore = "1.0.5" @@ -54,6 +55,7 @@ enable-ansi-support = "0.2.1" # we don't use the default formatter so we don't need default features env_logger = { version = "0.11.5", default-features = false } fixture-data = { path = "fixture-data" } +fs-err = "3.0.0" future-queue = "0.3.0" futures = "0.3.31" globset = "0.4.15" @@ -124,6 +126,7 @@ tracing = "0.1.41" tracing-subscriber = { version = "0.3.19", default-features = false, features = ["std", "tracing-log", "fmt"] } unicode-ident = "1.0.14" unicode-normalization = "0.1.24" +whoami = "1.5.2" win32job = "2.0.0" windows-sys = "0.59.0" winnow = "0.6.20" diff --git a/cargo-nextest/src/dispatch.rs b/cargo-nextest/src/dispatch.rs index 4b5235726b8..7d8cbaa472f 100644 --- a/cargo-nextest/src/dispatch.rs +++ b/cargo-nextest/src/dispatch.rs @@ -2099,6 +2099,9 @@ enum DebugCommand { #[arg(value_enum)] output_format: ExtractOutputFormat, }, + + /// Print the current executable path. + CurrentExe, } impl DebugCommand { @@ -2154,6 +2157,11 @@ impl DebugCommand { display_output_slice(output_slice, output_format)?; } } + DebugCommand::CurrentExe => { + let exe = std::env::current_exe() + .map_err(|err| ExpectedError::GetCurrentExeFailed { err })?; + println!("{}", exe.display()); + } } Ok(0) diff --git a/cargo-nextest/src/errors.rs b/cargo-nextest/src/errors.rs index 56dbb383011..7d27b98b8fc 100644 --- a/cargo-nextest/src/errors.rs +++ b/cargo-nextest/src/errors.rs @@ -32,6 +32,11 @@ pub enum ReuseBuildKind { pub enum ExpectedError { #[error("could not change to requested directory")] SetCurrentDirFailed { error: std::io::Error }, + #[error("failed to get current executable")] + GetCurrentExeFailed { + #[source] + err: std::io::Error, + }, #[error("cargo metadata exec failed")] CargoMetadataExecFailed { command: String, @@ -387,6 +392,7 @@ impl ExpectedError { Self::WorkspaceRootInvalidUtf8 { .. } | Self::WorkspaceRootInvalid { .. } | Self::SetCurrentDirFailed { .. } + | Self::GetCurrentExeFailed { .. } | Self::ProfileNotFound { .. } | Self::StoreDirCreateError { .. } | Self::RootManifestNotFound { .. } @@ -455,6 +461,10 @@ impl ExpectedError { error!("could not change to requested directory"); Some(error as &dyn Error) } + Self::GetCurrentExeFailed { err } => { + error!("failed to get current executable"); + Some(err as &dyn Error) + } Self::CargoMetadataExecFailed { command, err } => { error!("failed to execute `{}`", command.style(styles.bold)); Some(err as &dyn Error) diff --git a/fixture-data/src/nextest_tests.rs b/fixture-data/src/nextest_tests.rs index 1cddce9fd56..9997cb76377 100644 --- a/fixture-data/src/nextest_tests.rs +++ b/fixture-data/src/nextest_tests.rs @@ -24,8 +24,7 @@ pub static EXPECTED_TEST_SUITES: Lazy> vec![ TestCaseFixture::new("test_cargo_env_vars", TestCaseFixtureStatus::Pass) .with_property(TestCaseFixtureProperty::NotInDefaultSetUnix), - TestCaseFixture::new("test_cwd", TestCaseFixtureStatus::Pass) - .with_property(TestCaseFixtureProperty::NeedsSameCwd), + TestCaseFixture::new("test_cwd", TestCaseFixtureStatus::Pass), TestCaseFixture::new("test_execute_bin", TestCaseFixtureStatus::Pass), TestCaseFixture::new("test_failure_assert", TestCaseFixtureStatus::Fail), TestCaseFixture::new("test_failure_error", TestCaseFixtureStatus::Fail), diff --git a/fixtures/nextest-tests/.config/nextest.toml b/fixtures/nextest-tests/.config/nextest.toml index 4447c10937e..6f383e74746 100644 --- a/fixtures/nextest-tests/.config/nextest.toml +++ b/fixtures/nextest-tests/.config/nextest.toml @@ -82,6 +82,9 @@ default-filter = "not (test(test_flaky) | package(cdylib-example))" platform = 'cfg(unix)' default-filter = "not (test(test_flaky) | package(cdylib-example) | test(test_cargo_env_vars))" +[profile.archive-all] +archive.include = [{ path = "", relative-to = "target" }] + [test-groups.flaky] max-threads = 4 diff --git a/fixtures/nextest-tests/cdylib/cdylib-link/src/lib.rs b/fixtures/nextest-tests/cdylib/cdylib-link/src/lib.rs index 2e02892f0b3..fdd75b79632 100644 --- a/fixtures/nextest-tests/cdylib/cdylib-link/src/lib.rs +++ b/fixtures/nextest-tests/cdylib/cdylib-link/src/lib.rs @@ -8,5 +8,4 @@ extern "C" { #[test] fn test_multiply_two() { assert_eq!(unsafe { multiply_two(3, 3) }, 9); - } diff --git a/fixtures/nextest-tests/proc-macro-test/src/lib.rs b/fixtures/nextest-tests/proc-macro-test/src/lib.rs index e69de29bb2d..8b137891791 100644 --- a/fixtures/nextest-tests/proc-macro-test/src/lib.rs +++ b/fixtures/nextest-tests/proc-macro-test/src/lib.rs @@ -0,0 +1 @@ + diff --git a/fixtures/nextest-tests/tests/basic.rs b/fixtures/nextest-tests/tests/basic.rs index 5e92ea18d3f..73cca7c190c 100644 --- a/fixtures/nextest-tests/tests/basic.rs +++ b/fixtures/nextest-tests/tests/basic.rs @@ -1,9 +1,5 @@ // Copyright (c) The nextest Contributors -use std::{ - env, - io::Read, - path::{Path, PathBuf}, -}; +use std::{env, io::Read, path::PathBuf}; #[test] fn test_success() { @@ -66,10 +62,23 @@ fn test_failure_should_panic() {} #[test] fn test_cwd() { - // Ensure that the cwd is correct. + // Ensure that the cwd is correct. It's a bit tricky to do this in the face + // of a relative path, but just ensure that the cwd looks like what it + // should be (has a `Cargo.toml` with `name = "nextest-tests"` within it). let runtime_cwd = env::current_dir().expect("should be able to read current dir"); - let compile_time_cwd = Path::new(env!("CARGO_MANIFEST_DIR")); - assert_eq!(runtime_cwd, compile_time_cwd, "current dir matches"); + let cargo_toml_path = runtime_cwd.join("Cargo.toml"); + let cargo_toml = + std::fs::read_to_string(runtime_cwd.join("Cargo.toml")).unwrap_or_else(|error| { + panic!( + "should be able to read Cargo.toml: {}", + cargo_toml_path.display() + ) + }); + assert!( + cargo_toml.contains("name = \"nextest-tests\""), + "{} contains name = \"nextest-tests\"", + cargo_toml_path.display() + ); } #[test] @@ -149,10 +158,9 @@ fn test_cargo_env_vars() { // Note: we do not test CARGO here because nextest does not set it -- it's set by Cargo when // invoked as `cargo nextest`. - assert_env!( - "CARGO_MANIFEST_DIR", - "__NEXTEST_ORIGINAL_CARGO_MANIFEST_DIR" - ); + // Also, testing CARGO_MANIFEST_DIR is kind of tricky in the presence of the seed archive -- if we get it wrong, + // it will break in quite spectacular ways so don't bother. + assert_env!("CARGO_PKG_VERSION"); assert_env!("CARGO_PKG_VERSION_MAJOR"); assert_env!("CARGO_PKG_VERSION_MINOR"); diff --git a/integration-tests/Cargo.toml b/integration-tests/Cargo.toml index 281bd32ba71..7954d05894f 100644 --- a/integration-tests/Cargo.toml +++ b/integration-tests/Cargo.toml @@ -9,7 +9,14 @@ publish = false name = "cargo-nextest-dup" path = "test-helpers/cargo-nextest-dup.rs" +[[bin]] +name = "build-seed" +path = "test-helpers/build-seed.rs" + [dependencies] +camino.workspace = true +camino-tempfile.workspace = true + # We specify default-no-update here because if users just run: # # cargo build --no-default-features --features default-no-update @@ -18,20 +25,25 @@ path = "test-helpers/cargo-nextest-dup.rs" # We could ask distributors to always include `--package cargo-nextest` instead, but they're likely # to forget. None of our current tests depend on self-update, so just don't include it by default. cargo-nextest.workspace = true + color-eyre.workspace = true clap = { workspace = true, features = ["env"] } enable-ansi-support.workspace = true +fs-err.workspace = true +hex.workspace = true +nextest-metadata.workspace = true nextest-workspace-hack.workspace = true +serde_json.workspace = true +sha2.workspace = true +whoami.workspace = true [dev-dependencies] -camino-tempfile.workspace = true -camino.workspace = true cfg-if.workspace = true +cp_r.workspace = true fixture-data.workspace = true insta.workspace = true itertools.workspace = true nextest-metadata.workspace = true pathdiff.workspace = true regex.workspace = true -serde_json.workspace = true target-spec.workspace = true diff --git a/integration-tests/src/env.rs b/integration-tests/src/env.rs new file mode 100644 index 00000000000..3d530b5b444 --- /dev/null +++ b/integration-tests/src/env.rs @@ -0,0 +1,27 @@ +// Copyright (c) The nextest Contributors +// SPDX-License-Identifier: MIT OR Apache-2.0 + +#[track_caller] +pub fn set_env_vars() { + // The dynamic library tests require this flag. + std::env::set_var("RUSTFLAGS", "-C prefer-dynamic"); + // Set CARGO_TERM_COLOR to never to ensure that ANSI color codes don't interfere with the + // output. + // TODO: remove this once programmatic run statuses are supported. + std::env::set_var("CARGO_TERM_COLOR", "never"); + // This environment variable is required to test the #[bench] fixture. Note that THIS IS FOR + // TEST CODE ONLY. NEVER USE THIS IN PRODUCTION. + std::env::set_var("RUSTC_BOOTSTRAP", "1"); + + // Disable the tests which check for environment variables being set in `config.toml`, as they + // won't be in the search path when running integration tests. + std::env::set_var("__NEXTEST_NO_CHECK_CARGO_ENV_VARS", "1"); + + // Display empty STDOUT and STDERR lines in the output of failed tests. This + // allows tests which make sure outputs are being displayed to work. + std::env::set_var("__NEXTEST_DISPLAY_EMPTY_OUTPUTS", "1"); + + // Remove OUT_DIR from the environment, as it interferes with tests (some of them expect that + // OUT_DIR isn't set.) + std::env::remove_var("OUT_DIR"); +} diff --git a/integration-tests/src/lib.rs b/integration-tests/src/lib.rs new file mode 100644 index 00000000000..cf38c8558c7 --- /dev/null +++ b/integration-tests/src/lib.rs @@ -0,0 +1,6 @@ +// Copyright (c) The nextest Contributors +// SPDX-License-Identifier: MIT OR Apache-2.0 + +pub mod env; +pub mod nextest_cli; +pub mod seed; diff --git a/integration-tests/src/nextest_cli.rs b/integration-tests/src/nextest_cli.rs new file mode 100644 index 00000000000..5ff1ad9f767 --- /dev/null +++ b/integration-tests/src/nextest_cli.rs @@ -0,0 +1,177 @@ +// Copyright (c) The nextest Contributors +// SPDX-License-Identifier: MIT OR Apache-2.0 + +use camino::Utf8PathBuf; +use color_eyre::{ + eyre::{bail, Context}, + Result, +}; +use nextest_metadata::TestListSummary; +use std::{ + borrow::Cow, + collections::HashMap, + ffi::OsString, + fmt, + process::{Command, ExitStatus}, +}; + +pub fn cargo_bin() -> String { + match std::env::var("CARGO") { + Ok(v) => v, + Err(std::env::VarError::NotPresent) => "cargo".to_owned(), + Err(err) => panic!("error obtaining CARGO env var: {err}"), + } +} + +#[derive(Clone, Debug)] +pub struct CargoNextestCli { + bin: Utf8PathBuf, + args: Vec, + envs: HashMap, + unchecked: bool, +} + +impl CargoNextestCli { + pub fn new() -> Self { + let bin = std::env::var("NEXTEST_BIN_EXE_cargo-nextest-dup") + .expect("unable to find cargo-nextest-dup"); + Self { + bin: bin.into(), + args: vec!["nextest".to_owned()], + envs: HashMap::new(), + unchecked: false, + } + } + + /// Creates a new CargoNextestCli instance for use in a setup script. + /// + /// Scripts don't have access to the `NEXTEST_BIN_EXE_cargo-nextest-dup` environment variable, + /// so we run `cargo run --bin cargo-nextest-dup nextest debug current-exe` instead. + pub fn new_for_script() -> Result { + let cargo_bin = cargo_bin(); + let mut command = std::process::Command::new(&cargo_bin); + command.args(&[ + "run", + "--bin", + "cargo-nextest-dup", + "--", + "nextest", + "debug", + "current-exe", + ]); + let output = command.output().wrap_err("failed to get current exe")?; + + let output = CargoNextestOutput { + command, + exit_status: output.status, + stdout: output.stdout, + stderr: output.stderr, + }; + + if !output.exit_status.success() { + bail!("failed to get current exe:\n\n{output:?}"); + } + + // The output is the path to the current exe. + let exe = + String::from_utf8(output.stdout).wrap_err("current exe output isn't valid UTF-8")?; + + Ok(Self { + bin: Utf8PathBuf::from(exe.trim_end()), + args: vec!["nextest".to_owned()], + envs: HashMap::new(), + unchecked: false, + }) + } + + pub fn arg(&mut self, arg: impl Into) -> &mut Self { + self.args.push(arg.into()); + self + } + + pub fn args(&mut self, arg: impl IntoIterator>) -> &mut Self { + self.args.extend(arg.into_iter().map(Into::into)); + self + } + + pub fn env(&mut self, k: impl Into, v: impl Into) -> &mut Self { + self.envs.insert(k.into(), v.into()); + self + } + + pub fn envs( + &mut self, + envs: impl IntoIterator, impl Into)>, + ) -> &mut Self { + self.envs + .extend(envs.into_iter().map(|(k, v)| (k.into(), v.into()))); + self + } + + pub fn unchecked(&mut self, unchecked: bool) -> &mut Self { + self.unchecked = unchecked; + self + } + + pub fn output(&self) -> CargoNextestOutput { + let mut command = std::process::Command::new(&self.bin); + command.args(&self.args); + command.envs(&self.envs); + let output = command.output().expect("failed to execute"); + + let ret = CargoNextestOutput { + command, + exit_status: output.status, + stdout: output.stdout, + stderr: output.stderr, + }; + + if !self.unchecked && !output.status.success() { + panic!("command failed:\n\n{ret}"); + } + + ret + } +} + +pub struct CargoNextestOutput { + pub command: Command, + pub exit_status: ExitStatus, + pub stdout: Vec, + pub stderr: Vec, +} + +impl CargoNextestOutput { + pub fn stdout_as_str(&self) -> Cow<'_, str> { + String::from_utf8_lossy(&self.stdout) + } + + pub fn stderr_as_str(&self) -> Cow<'_, str> { + String::from_utf8_lossy(&self.stderr) + } + + pub fn decode_test_list_json(&self) -> Result { + Ok(serde_json::from_slice(&self.stdout)?) + } +} + +impl fmt::Display for CargoNextestOutput { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!( + f, + "command: {:?}\nexit code: {:?}\n\ + --- stdout ---\n{}\n\n--- stderr ---\n{}\n\n", + self.command, + self.exit_status.code(), + String::from_utf8_lossy(&self.stdout), + String::from_utf8_lossy(&self.stderr) + ) + } +} + +// Make Debug output the same as Display output, so `.unwrap()` and `.expect()` are nicer. +impl fmt::Debug for CargoNextestOutput { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + fmt::Display::fmt(self, f) + } +} diff --git a/integration-tests/src/seed.rs b/integration-tests/src/seed.rs new file mode 100644 index 00000000000..64aa77d5d0d --- /dev/null +++ b/integration-tests/src/seed.rs @@ -0,0 +1,127 @@ +// Copyright (c) The nextest Contributors +// SPDX-License-Identifier: MIT OR Apache-2.0 + +use crate::{env::set_env_vars, nextest_cli::CargoNextestCli}; +use camino::{Utf8Path, Utf8PathBuf}; +use color_eyre::eyre::Context; +use fs_err as fs; +use sha2::{Digest, Sha256}; +use std::{collections::BTreeMap, io}; + +pub fn nextest_tests_dir() -> Utf8PathBuf { + Utf8Path::new(env!("CARGO_MANIFEST_DIR")) + .parent() + .unwrap() + .join("fixtures/nextest-tests") +} + +// We use SHA-256 because other parts of nextest do the same -- this can easily +// be changed to another hash function if needed. +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] +pub struct Sha256Hash([u8; 32]); + +impl std::fmt::Display for Sha256Hash { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + hex::encode(&self.0).fmt(f) + } +} + +pub fn compute_dir_hash(dir: impl AsRef) -> io::Result { + let hashes = hash_all_files(dir.as_ref(), true)?; + let mut hasher = Sha256::new(); + for (file_name, hash) in hashes { + hasher.update(file_name.as_str().as_bytes()); + hasher.update(&[0]); + hasher.update(&hash.0); + hasher.update(&[0]); + } + Ok(Sha256Hash(hasher.finalize().into())) +} + +// Hash all the files in a directory. +// +// Using a `BTreeMap` ensures a deterministic order of files above. +pub(super) fn hash_all_files( + dir: &Utf8Path, + root: bool, +) -> io::Result> { + let mut stack = vec![dir.to_path_buf()]; + let mut hashes = BTreeMap::new(); + + // TODO: parallelize this? + while let Some(dir) = stack.pop() { + for entry in dir.read_dir_utf8()? { + let entry = entry?; + let ty = entry.file_type()?; + + // Ignore a pre-existing `target` directory at the root. + if root && entry.path().file_name() == Some("target") { + continue; + } + + if ty.is_dir() { + stack.push(entry.into_path()); + } else if ty.is_file() { + let path = entry.into_path(); + let contents = fs::read(&path)?; + let hash = Sha256Hash(Sha256::digest(&contents).into()); + hashes.insert(path, hash); + } + } + } + + Ok(hashes) +} + +pub fn get_seed_archive_name(hash: Sha256Hash) -> Utf8PathBuf { + // Check in the std temp directory for the seed file. + let temp_dir = Utf8PathBuf::try_from(std::env::temp_dir()).expect("temp dir is utf-8"); + let username = whoami::username(); + let user_dir = temp_dir.join(format!("nextest-tests-seed-{username}")); + user_dir.join(format!("seed-{hash}.tar.zst")) +} + +pub fn make_seed_archive(workspace_dir: &Utf8Path, file_name: &Utf8Path) -> color_eyre::Result<()> { + // Make the directory containing the file name. + fs::create_dir_all(file_name.parent().unwrap())?; + + // First, run a build in a temporary directory. + let temp_dir = camino_tempfile::Builder::new() + .prefix("nextest-seed-build-") + .tempdir() + .wrap_err("failed to create temporary directory")?; + let target_dir = temp_dir.path().join("target"); + fs::create_dir_all(&target_dir)?; + + // Now build a nextest archive, using the temporary directory as the target dir. + let mut cli = CargoNextestCli::new_for_script()?; + + // Set the environment variables after getting the CLI -- this avoids + // rebuilds due to the variables. + // + // TODO: this should be part of nextest_cli.rs. + set_env_vars(); + + let output = cli + .args([ + "--manifest-path", + workspace_dir.join("Cargo.toml").as_str(), + "archive", + "--archive-file", + file_name.as_str(), + "--workspace", + "--all-targets", + "--target-dir", + target_dir.as_str(), + // Use this profile to ensure that the entire target dir is included. + "--profile", + "archive-all", + ]) + .output(); + + if std::env::var("INTEGRATION_TESTS_DEBUG") == Ok("1".to_string()) { + eprintln!("make_seed_archive output: {output}"); + } + + Ok(()) +} diff --git a/integration-tests/test-helpers/build-seed.rs b/integration-tests/test-helpers/build-seed.rs new file mode 100644 index 00000000000..d8108cc4a33 --- /dev/null +++ b/integration-tests/test-helpers/build-seed.rs @@ -0,0 +1,40 @@ +// Copyright (c) The nextest Contributors +// SPDX-License-Identifier: MIT OR Apache-2.0 + +//! A fixture to build a copy of nextest-tests and prepare it for testing. + +use color_eyre::eyre::Context; +use integration_tests::seed::{ + compute_dir_hash, get_seed_archive_name, make_seed_archive, nextest_tests_dir, +}; + +fn main() -> color_eyre::Result<()> { + color_eyre::install()?; + + let tests_dir = nextest_tests_dir(); + let nextest_env_file = std::env::var("NEXTEST_ENV") + .wrap_err("unable to find NEXTEST_ENV -- is this being run as a setup script?")?; + + // First, hash the nextest-tests fixture. + let dir_hash = compute_dir_hash(&tests_dir)?; + + // Get the seed archive name. + let seed_archive_name = get_seed_archive_name(dir_hash); + + // Does the seed file exist? + if seed_archive_name.is_file() { + println!("info: using existing seed archive: {seed_archive_name}"); + } else { + // Otherwise, create a new seed archive. + println!("info: unable to find seed archive {seed_archive_name}, building"); + make_seed_archive(&tests_dir, &seed_archive_name)?; + println!("info: created new seed archive: {seed_archive_name}"); + } + + fs_err::write( + &nextest_env_file, + format!("SEED_ARCHIVE={seed_archive_name}"), + )?; + + Ok(()) +} diff --git a/integration-tests/tests/integration/fixtures.rs b/integration-tests/tests/integration/fixtures.rs index 98a3ffe1cc0..dfa187b80cf 100644 --- a/integration-tests/tests/integration/fixtures.rs +++ b/integration-tests/tests/integration/fixtures.rs @@ -2,170 +2,16 @@ // SPDX-License-Identifier: MIT OR Apache-2.0 use super::temp_project::TempProject; -use camino::Utf8PathBuf; -use color_eyre::Result; use fixture_data::{ models::{TestCaseFixtureProperty, TestCaseFixtureStatus, TestSuiteFixtureProperty}, nextest_tests::EXPECTED_TEST_SUITES, }; +use integration_tests::nextest_cli::{cargo_bin, CargoNextestCli}; use nextest_metadata::{ BinaryListSummary, BuildPlatform, RustTestSuiteStatusSummary, TestListSummary, }; use regex::Regex; -use std::{ - borrow::Cow, - collections::HashMap, - ffi::OsString, - fmt, - process::{Command, ExitStatus}, -}; - -pub fn cargo_bin() -> String { - match std::env::var("CARGO") { - Ok(v) => v, - Err(std::env::VarError::NotPresent) => "cargo".to_owned(), - Err(err) => panic!("error obtaining CARGO env var: {err}"), - } -} - -#[derive(Clone, Debug)] -pub struct CargoNextestCli { - bin: Utf8PathBuf, - args: Vec, - envs: HashMap, - unchecked: bool, -} - -impl CargoNextestCli { - pub fn new() -> Self { - let bin = std::env::var("NEXTEST_BIN_EXE_cargo-nextest-dup") - .expect("unable to find cargo-nextest-dup"); - Self { - bin: bin.into(), - args: Vec::new(), - envs: HashMap::new(), - unchecked: false, - } - } - - #[expect(dead_code)] - pub fn arg(&mut self, arg: impl Into) -> &mut Self { - self.args.push(arg.into()); - self - } - - pub fn args(&mut self, arg: impl IntoIterator>) -> &mut Self { - self.args.extend(arg.into_iter().map(Into::into)); - self - } - - pub fn env(&mut self, k: impl Into, v: impl Into) -> &mut Self { - self.envs.insert(k.into(), v.into()); - self - } - - #[expect(dead_code)] - pub fn envs( - &mut self, - envs: impl IntoIterator, impl Into)>, - ) -> &mut Self { - self.envs - .extend(envs.into_iter().map(|(k, v)| (k.into(), v.into()))); - self - } - - pub fn unchecked(&mut self, unchecked: bool) -> &mut Self { - self.unchecked = unchecked; - self - } - - pub fn output(&self) -> CargoNextestOutput { - let mut command = std::process::Command::new(&self.bin); - command.arg("nextest").args(&self.args); - command.envs(&self.envs); - let output = command.output().expect("failed to execute"); - - let ret = CargoNextestOutput { - command, - exit_status: output.status, - stdout: output.stdout, - stderr: output.stderr, - }; - - if !self.unchecked && !output.status.success() { - panic!("command failed:\n\n{ret}"); - } - - ret - } -} - -pub struct CargoNextestOutput { - pub command: Command, - pub exit_status: ExitStatus, - pub stdout: Vec, - pub stderr: Vec, -} - -impl CargoNextestOutput { - pub fn stdout_as_str(&self) -> Cow<'_, str> { - String::from_utf8_lossy(&self.stdout) - } - - pub fn stderr_as_str(&self) -> Cow<'_, str> { - String::from_utf8_lossy(&self.stderr) - } - - pub fn decode_test_list_json(&self) -> Result { - Ok(serde_json::from_slice(&self.stdout)?) - } -} - -impl fmt::Display for CargoNextestOutput { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!( - f, - "command: {:?}\nexit code: {:?}\n\ - --- stdout ---\n{}\n\n--- stderr ---\n{}\n\n", - self.command, - self.exit_status.code(), - String::from_utf8_lossy(&self.stdout), - String::from_utf8_lossy(&self.stderr) - ) - } -} - -// Make Debug output the same as Display output, so `.unwrap()` and `.expect()` are nicer. -impl fmt::Debug for CargoNextestOutput { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - fmt::Display::fmt(self, f) - } -} - -#[track_caller] -pub(super) fn set_env_vars() { - // The dynamic library tests require this flag. - std::env::set_var("RUSTFLAGS", "-C prefer-dynamic"); - // Set CARGO_TERM_COLOR to never to ensure that ANSI color codes don't interfere with the - // output. - // TODO: remove this once programmatic run statuses are supported. - std::env::set_var("CARGO_TERM_COLOR", "never"); - // This environment variable is required to test the #[bench] fixture. Note that THIS IS FOR - // TEST CODE ONLY. NEVER USE THIS IN PRODUCTION. - std::env::set_var("RUSTC_BOOTSTRAP", "1"); - - // Disable the tests which check for environment variables being set in `config.toml`, as they - // won't be in the search path when running integration tests. - std::env::set_var("__NEXTEST_NO_CHECK_CARGO_ENV_VARS", "1"); - - // Display empty STDOUT and STDERR lines in the output of failed tests. This - // allows tests which make sure outputs are being displayed to work. - std::env::set_var("__NEXTEST_DISPLAY_EMPTY_OUTPUTS", "1"); - - // Remove OUT_DIR from the environment, as it interferes with tests (some of them expect that - // OUT_DIR isn't set.) - std::env::remove_var("OUT_DIR"); -} +use std::process::Command; #[track_caller] pub fn save_cargo_metadata(p: &TempProject) { diff --git a/integration-tests/tests/integration/main.rs b/integration-tests/tests/integration/main.rs index 1464abda7aa..8d8bfb651f1 100644 --- a/integration-tests/tests/integration/main.rs +++ b/integration-tests/tests/integration/main.rs @@ -23,6 +23,10 @@ //! `NEXTEST_BIN_EXE_cargo-nextest-dup`. use camino::{Utf8Path, Utf8PathBuf}; +use integration_tests::{ + env::set_env_vars, + nextest_cli::{CargoNextestCli, CargoNextestOutput}, +}; use nextest_metadata::{BuildPlatform, NextestExitCode, TestListSummary}; use std::{borrow::Cow, fs::File, io::Write}; use target_spec::Platform; @@ -130,6 +134,8 @@ fn test_list_full() { "json", "--list-type", "full", + "--cargo-verbose", + "--cargo-verbose", ]) .output(); @@ -623,21 +629,20 @@ fn test_relocated_run() { set_env_vars(); let custom_target_dir = Utf8TempDir::new().unwrap(); - let custom_target_path = custom_target_dir.path(); - let p = TempProject::new_custom_target_dir(custom_target_path).unwrap(); - + let custom_target_path = custom_target_dir.path().join("target"); + let p = TempProject::new_custom_target_dir(&custom_target_path).unwrap(); save_binaries_metadata(&p); save_cargo_metadata(&p); let mut p2 = TempProject::new().unwrap(); - let new_target_path = p2.workspace_root().join("test-subdir"); + let new_target_path = p2.workspace_root().join("test-subdir/target"); // copy target directory over std::fs::create_dir_all(&new_target_path).unwrap(); - temp_project::copy_dir_all(custom_target_path, &new_target_path, false).unwrap(); + temp_project::copy_dir_all(&custom_target_path, &new_target_path, false).unwrap(); // Remove the old target path to ensure that any tests that refer to files within it // fail. - std::fs::remove_dir_all(custom_target_path).unwrap(); + std::fs::remove_dir_all(&custom_target_path).unwrap(); p2.set_target_dir(new_target_path); @@ -769,8 +774,8 @@ fn create_archive( snapshot_name: &str, ) -> Result<(TempProject, Utf8PathBuf), CargoNextestOutput> { let custom_target_dir = Utf8TempDir::new().unwrap(); - let custom_target_path = custom_target_dir.path(); - let p = TempProject::new_custom_target_dir(custom_target_path).unwrap(); + let custom_target_path = custom_target_dir.path().join("target"); + let p = TempProject::new_custom_target_dir(&custom_target_path).unwrap(); let config_path = p.workspace_root().join(".config/nextest.toml"); std::fs::write(config_path, config_contents).unwrap(); diff --git a/integration-tests/tests/integration/snapshots/integration__archive_includes.snap b/integration-tests/tests/integration/snapshots/integration__archive_includes.snap index f37403d2b76..1e0e6e9a9c4 100644 --- a/integration-tests/tests/integration/snapshots/integration__archive_includes.snap +++ b/integration-tests/tests/integration/snapshots/integration__archive_includes.snap @@ -1,13 +1,12 @@ --- source: integration-tests/tests/integration/main.rs expression: output.stderr_as_str() +snapshot_kind: text --- - Archiving 17 binaries (including 2 non-test binaries), 2 build script output directories, 2 linked paths, 7 extra paths, and 1 standard library to + Archiving 17 binaries (including 2 non-test binaries), 2 build script output directories, 1 linked path, 7 extra paths, and 1 standard library to Warning ignoring extra path `/excluded-dir` because it does not exist Warning ignoring extra path `/depth-0-dir` specified with depth 0 since it is a directory Warning ignoring extra path `/file_that_does_not_exist.txt` because it does not exist Warning while archiving extra paths, ignoring `/uds.sock` because it is not a file, directory, or symbolic link - Warning linked path `/debug/build//does-not-exist` not found, requested by: cdylib-link v0.1.0 - (this is a bug in this crate that should be fixed) Warning while archiving extra paths, recursion depth exceeded at /application-data/d1/d2/d3/d4/d5/d6/d7/d8/d9/d10/d11/d12/d13/d14/d15/d16 (limit: 16) Archived files to in diff --git a/integration-tests/tests/integration/snapshots/integration__archive_includes_without_uds.snap b/integration-tests/tests/integration/snapshots/integration__archive_includes_without_uds.snap index 1a0a428de33..0fd5e37cef5 100644 --- a/integration-tests/tests/integration/snapshots/integration__archive_includes_without_uds.snap +++ b/integration-tests/tests/integration/snapshots/integration__archive_includes_without_uds.snap @@ -2,12 +2,10 @@ source: integration-tests/tests/integration/main.rs expression: output.stderr_as_str() --- - Archiving 17 binaries (including 2 non-test binaries), 2 build script output directories, 2 linked paths, 7 extra paths, and 1 standard library to + Archiving 17 binaries (including 2 non-test binaries), 2 build script output directories, 1 linked path, 7 extra paths, and 1 standard library to Warning ignoring extra path `/excluded-dir` because it does not exist Warning ignoring extra path `/depth-0-dir` specified with depth 0 since it is a directory Warning ignoring extra path `/file_that_does_not_exist.txt` because it does not exist Warning ignoring extra path `/uds.sock` because it does not exist - Warning linked path `/debug/build//does-not-exist` not found, requested by: cdylib-link v0.1.0 - (this is a bug in this crate that should be fixed) Warning while archiving extra paths, recursion depth exceeded at /application-data/d1/d2/d3/d4/d5/d6/d7/d8/d9/d10/d11/d12/d13/d14/d15/d16 (limit: 16) Archived files to in diff --git a/integration-tests/tests/integration/snapshots/integration__archive_missing_includes.snap b/integration-tests/tests/integration/snapshots/integration__archive_missing_includes.snap index 894731c41b9..986f3d4f8c1 100644 --- a/integration-tests/tests/integration/snapshots/integration__archive_missing_includes.snap +++ b/integration-tests/tests/integration/snapshots/integration__archive_missing_includes.snap @@ -1,8 +1,9 @@ --- source: integration-tests/tests/integration/main.rs expression: output.stderr_as_str() +snapshot_kind: text --- - Archiving 17 binaries (including 2 non-test binaries), 2 build script output directories, 2 linked paths, 1 extra path, and 1 standard library to + Archiving 17 binaries (including 2 non-test binaries), 2 build script output directories, 1 linked path, 1 extra path, and 1 standard library to error: error creating archive `` Caused by: diff --git a/integration-tests/tests/integration/snapshots/integration__archive_no_includes.snap b/integration-tests/tests/integration/snapshots/integration__archive_no_includes.snap index 6d262313979..5f0c3318488 100644 --- a/integration-tests/tests/integration/snapshots/integration__archive_no_includes.snap +++ b/integration-tests/tests/integration/snapshots/integration__archive_no_includes.snap @@ -1,8 +1,7 @@ --- source: integration-tests/tests/integration/main.rs expression: output.stderr_as_str() +snapshot_kind: text --- - Archiving 17 binaries (including 2 non-test binaries), 2 build script output directories, 2 linked paths, and 1 standard library to - Warning linked path `/debug/build//does-not-exist` not found, requested by: cdylib-link v0.1.0 - (this is a bug in this crate that should be fixed) + Archiving 17 binaries (including 2 non-test binaries), 2 build script output directories, 1 linked path, and 1 standard library to Archived files to in diff --git a/integration-tests/tests/integration/temp_project.rs b/integration-tests/tests/integration/temp_project.rs index d4aed868c62..b4f3f32a5c9 100644 --- a/integration-tests/tests/integration/temp_project.rs +++ b/integration-tests/tests/integration/temp_project.rs @@ -3,7 +3,11 @@ use camino::{Utf8Path, Utf8PathBuf}; use camino_tempfile::Utf8TempDir; -use std::{fs, io, path::Path}; +use color_eyre::eyre::{bail, Context}; +use cp_r::CopyOptions; +use fs_err as fs; +use integration_tests::{nextest_cli::CargoNextestCli, seed::nextest_tests_dir}; +use std::path::Path; // This isn't general purpose -- it specifically excludes certain directories at the root and is // generally not race-free. @@ -11,23 +15,20 @@ pub(super) fn copy_dir_all( src: impl AsRef, dst: impl AsRef, root: bool, -) -> io::Result<()> { +) -> color_eyre::Result<()> { let src = src.as_ref(); let dst = dst.as_ref(); fs::create_dir_all(dst)?; - for entry in fs::read_dir(src)? { - let entry = entry?; - let ty = entry.file_type()?; - if ty.is_dir() { - if root && entry.path().file_name() == Some(std::ffi::OsStr::new("target")) { - continue; + // Do a copy, preserving mtimes -- this ensures that the seed is used. + CopyOptions::new() + .filter(|path, _| { + if !root { + return Ok(true); } - copy_dir_all(entry.path(), dst.join(entry.file_name()), false)?; - } else { - fs::copy(entry.path(), dst.join(entry.file_name()))?; - } - } + Ok(path.as_os_str() != "target") + }) + .copy_tree(src, dst)?; Ok(()) } @@ -52,17 +53,21 @@ impl TempProject { } fn new_impl(custom_target_dir: Option) -> color_eyre::Result { + // Ensure that a custom target dir, if specified, ends with "target". + if let Some(dir) = &custom_target_dir { + if !dir.ends_with("target") { + bail!("custom target directory must end with 'target'"); + } + } + let temp_dir = camino_tempfile::Builder::new() - .prefix("nextest-fixture") + .prefix("nextest-fixture-") .tempdir()?; // Note: can't use canonicalize here because it ends up creating a UNC path on Windows, // which doesn't match compile time. let temp_root: Utf8PathBuf = fixup_macos_path(temp_dir.path()); let workspace_root = temp_root.join("src"); - let src_dir = Utf8Path::new(env!("CARGO_MANIFEST_DIR")) - .parent() - .unwrap() - .join("fixtures/nextest-tests"); + let src_dir = nextest_tests_dir(); copy_dir_all(src_dir, &workspace_root, true)?; @@ -71,12 +76,30 @@ impl TempProject { None => workspace_root.join("target"), }; - Ok(Self { + let ret = Self { temp_dir: Some(temp_dir), temp_root, workspace_root, target_dir, - }) + }; + + // Extract the seed into the target directory. + let seed_archive = std::env::var("SEED_ARCHIVE") + .wrap_err("SEED_ARCHIVE not set -- the setup script should have set it")?; + _ = CargoNextestCli::new() + .args([ + "run", + "--no-run", + "--manifest-path", + ret.manifest_path().as_str(), + "--archive-file", + seed_archive.as_str(), + "--extract-to", + ret.target_dir().parent().unwrap().as_str(), + ]) + .output(); + + Ok(ret) } #[expect(dead_code)] @@ -99,7 +122,11 @@ impl TempProject { } pub fn set_target_dir(&mut self, target_dir: impl Into) { - self.target_dir = target_dir.into(); + let target_dir = target_dir.into(); + if !target_dir.ends_with("target") { + panic!("custom target directory `{target_dir}` must end with 'target'"); + } + self.target_dir = target_dir; } pub fn binaries_metadata_path(&self) -> Utf8PathBuf {