tests: Split up tmt tests into separate plans

Because tmt right now doesn't support isolation automatically,
and many of the tests we'll want to write mutate the system,
we'll need to copy-paste the base plan.

This changes the `make test-tmt` to have its own little
tmt wrapper that dynamically scans the plans/ and sorts
them by a priority, then runs them one by one currently.

Closes: containers#735

Signed-off-by: Colin Walters <[email protected]>

plans/integration-run.fmf → plans/test-01-readonly.fmf

@@ -5,14 +5,18 @@ provision:
   # Generated by make test-tmt
   image: file://./target/testvm/disk.qcow2
   disk: 20
-summary: Execute booted tests
+summary: Execute booted readonly/nondestructive tests
 execute:
   how: tmt
   # There's currently two dynamic test frameworks; python and nushell.
   # python is well known and understood. nushell is less well known, but
   # is quite nice for running subprocesses and the like while making
   # it easy to parse JSON etc.
+  # All of these tests should generally be read-only - avoid any kind
+  # of persistent changes.
+  # If you need to do that, unfortunately right now that needs to be
+  # a separate plan.
   script: |
     set -xeu
-    pytest tests/booted/*.py
-    ls tests/booted/*-test-*.nu |sort -n | while read t; do nu $t; done
+    pytest tests/booted/readonly/*.py
+    ls tests/booted/readonly/*-test-*.nu |sort -n | while read t; do nu $t; done

plans/test-20-local-upgrade.fmf

@@ -0,0 +1,12 @@
+# 
+provision:
+  how: virtual
+  # Generated by make test-tmt
+  image: file://./target/testvm/disk.qcow2
+  disk: 20
+summary: Execute local upgrade tests
+execute:
+  how: tmt
+  # We avoid writing nontrivial shell script as a general rule,
+  # so this is written in nu.
+  script: exec nu tests/booted/test-image-pushpull-upgrade.nu

tests/booted/readonly/tap.nu

@@ -0,0 +1 @@
+../tap.nu

xtask/src/xtask.rs

@@ -148,13 +148,48 @@ fn update_generated(sh: &Shell) -> Result<()> {
 
 #[context("test-integration")]
 fn test_tmt(sh: &Shell) -> Result<()> {
+    // We need to split most of our tests into separate plans because tmt doesn't
+    // support automatic isolation. (xref)
+    let mut all_plan_files =
+        sh.read_dir("plans")?
+            .into_iter()
+            .try_fold(Vec::new(), |mut acc, ent| -> Result<_> {
+                let path = Utf8PathBuf::try_from(ent)?;
+                let Some(ext) = path.extension() else {
+                    return Ok(acc);
+                };
+                if ext != "fmf" {
+                    return Ok(acc);
+                }
+                let stem = path.file_stem().expect("file stem");
+                let Some((prefix, suffix)) = stem.split_once('-') else {
+                    return Ok(acc);
+                };
+                if prefix != "test" {
+                    return Ok(acc);
+                }
+                let Some((priority, _)) = suffix.split_once('-') else {
+                    anyhow::bail!("Invalid test {path}");
+                };
+                let priority: u32 = priority
+                    .parse()
+                    .with_context(|| format!("Parsing {path}"))?;
+                acc.push((priority, stem.to_string()));
+                Ok(acc)
+            })?;
+    all_plan_files.sort_by_key(|v| v.0);
+    println!("Discovered plans: {all_plan_files:?}");
+
     cmd!(sh, "cargo run -p tests-integration run-vm prepare-tmt").run()?;
     // cc https://pagure.io/testcloud/pull-request/174
     cmd!(sh, "rm -vf /var/tmp/tmt/testcloud/images/disk.qcow2").run()?;
-    if let Err(e) = cmd!(sh, "tmt run plans -n integration-run").run() {
-        // tmt annoyingly does not output errors by default
-        let _ = cmd!(sh, "tmt run -l report -vvv").run();
-        return Err(e.into());
+
+    for (_prio, name) in all_plan_files {
+        if let Err(e) = cmd!(sh, "tmt run plans -n {name}").run() {
+            // tmt annoyingly does not output errors by default
+            let _ = cmd!(sh, "tmt run -l report -vvv").run();
+            return Err(e.into());
+        }
     }
     Ok(())
 }