wip: Install with fsverity

Signed-off-by: Colin Walters <[email protected]>

.github/workflows/ci.yml

@@ -71,10 +71,17 @@ jobs:
         run: sudo rm -f /bin/skopeo /usr/bin/skopeo
       - name: Free up disk space on runner
         run: sudo ./ci/clean-gha-runner.sh
+      - name: Enable fsverity for /
+        run: sudo tune2fs -O verity $(findmnt -vno SOURCE /)
+      - name: Install utils
+        run: sudo apt -y install fsverity
       - name: Integration tests
         run: |
           set -xeu
+          # Build images to test; TODO investigate doing single container builds
+          # via GHA and pushing to a temporary registry to share among workflows?
           sudo podman build -t localhost/bootc -f hack/Containerfile .
+          sudo podman build -t localhost/bootc-fsverity -f ci/Containerfile.install-fsverity
           export CARGO_INCREMENTAL=0  # because we aren't caching the test runner bits
           cargo build --release -p tests-integration
           df -h /
@@ -83,8 +90,9 @@ jobs:
           df -h /
           # Nondestructive but privileged tests
           sudo bootc-integration-tests host-privileged localhost/bootc
-          # Finally the install-alongside suite
+          # Install tests
           sudo bootc-integration-tests install-alongside localhost/bootc
+          sudo bootc-integration-tests install-fsverity localhost/bootc-fsverity
   docs:
     if: ${{ contains(github.event.pull_request.labels.*.name, 'documentation') }}
     runs-on: ubuntu-latest

ci/Containerfile.install-fsverity

@@ -0,0 +1,10 @@
+# Enable fsverity at install time
+FROM localhost/bootc
+RUN <<EORUN
+set -xeuo pipefail
+mkdir -p /usr/lib/bootc/install
+cat > /usr/lib/bootc/install/30-fsverity.toml <<EOF
+[install]
+fsverity = "enabled"
+EOF
+EORUN

lib/Cargo.toml

@@ -23,6 +23,8 @@ ostree-ext = { path = "../ostree-ext" }
 chrono = { workspace = true, features = ["serde"] }
 clap = { workspace = true, features = ["derive","cargo"] }
 clap_mangen = { workspace = true, optional = true }
+#composefs = "0.2.0"
+composefs = { path = "../../composefs-rs" }
 cap-std-ext = { workspace = true, features = ["fs_utf8"] }
 hex = { workspace = true }
 fn-error-context = { workspace = true }

lib/src/cli.rs

@@ -280,6 +280,16 @@ pub(crate) enum ImageOpts {
     Cmd(ImageCmdOpts),
 }
 
+/// Options for consistency checking
+#[derive(Debug, clap::Subcommand, PartialEq, Eq)]
+pub(crate) enum FsckOpts {
+    /// Check the state of fsverity on the ostree objects. Possible output:
+    /// "enabled" => All .file objects have fsverity
+    /// "disabled" => No .file objects have fsverity
+    /// "inconsistent" => Mixed state
+    OstreeVerity,
+}
+
 /// Hidden, internal only options
 #[derive(Debug, clap::Subcommand, PartialEq, Eq)]
 pub(crate) enum InternalsOpts {
@@ -293,6 +303,8 @@ pub(crate) enum InternalsOpts {
     FixupEtcFstab,
     /// Should only be used by `make update-generated`
     PrintJsonSchema,
+    /// Perform consistency checking.
+    Fsck,
     /// Perform cleanup actions
     Cleanup,
     /// Proxy frontend for the `ostree-ext` CLI.
@@ -952,6 +964,20 @@ async fn run_from_opt(opt: Opt) -> Result<()> {
                 )
                 .await
             }
+            InternalsOpts::Fsck => {
+                let storage = get_storage().await?;
+                let r = crate::fsck::fsck(&storage).await?;
+                match r.errors.as_slice() {
+                    [] => {}
+                    errs => {
+                        for err in errs {
+                            eprintln!("error: {err}");
+                        }
+                        anyhow::bail!("fsck found errors");
+                    }
+                }
+                Ok(())
+            }
             InternalsOpts::FixupEtcFstab => crate::deploy::fixup_etc_fstab(&root),
             InternalsOpts::PrintJsonSchema => {
                 let schema = schema_for!(crate::spec::Host);

lib/src/fsck.rs

@@ -0,0 +1,127 @@
+//! # Write deployments merging image with configmap
+//!
+//! Create a merged filesystem tree with the image and mounted configmaps.
+
+use std::os::fd::AsFd;
+use std::str::FromStr as _;
+
+use anyhow::Ok;
+use anyhow::{Context, Result};
+use camino::Utf8PathBuf;
+use cap_std::fs::Dir;
+use cap_std_ext::cap_std;
+use fn_error_context::context;
+use ostree_ext::keyfileext::KeyFileExt;
+use ostree_ext::ostree;
+use rustix::io::Errno;
+use serde::{Deserialize, Serialize};
+
+use crate::install::config::Tristate;
+use crate::store::{self, Storage};
+
+#[derive(Serialize, Deserialize, Debug, PartialEq, Eq)]
+#[serde(rename_all = "kebab-case")]
+pub(crate) enum VerityState {
+    Enabled,
+    Disabled,
+    Inconsistent,
+}
+
+#[derive(Default, Serialize, Deserialize, Debug, PartialEq, Eq)]
+pub(crate) struct FsckResult {
+    pub(crate) errors: Vec<String>,
+    pub(crate) verity: Option<VerityState>,
+}
+
+/// Check the fsverity state of all regular files in this object directory.
+#[context("Computing verity state")]
+fn verity_state_of_objects(d: &Dir) -> Result<(u64, u64)> {
+    let mut enabled = 0;
+    let mut disabled = 0;
+    for ent in d.entries()? {
+        let ent = ent?;
+        if !ent.file_type()?.is_file() {
+            continue;
+        }
+        let name = ent.file_name();
+        let name = name
+            .into_string()
+            .map(Utf8PathBuf::from)
+            .map_err(|_| anyhow::anyhow!("Invalid UTF-8"))?;
+        let Some("file") = name.extension() else {
+            continue;
+        };
+        let f = d
+            .open(&name)
+            .with_context(|| format!("Failed to open {name}"))?;
+        let r: Option<composefs::fsverity::Sha256HashValue> =
+            match composefs::fsverity::ioctl::fs_ioc_measure_verity(f.as_fd()) {
+                Ok(r) => Some(r),
+                Err(e) if matches!(e.downcast_ref::<Errno>(), Some(&Errno::NOSYS)) => None,
+                Err(e) => return Err(e),
+            };
+        drop(f);
+        if r.is_some() {
+            enabled += 1;
+        } else {
+            disabled += 1;
+        }
+    }
+    Ok((enabled, disabled))
+}
+
+async fn verity_state_of_all_objects(repo: &ostree::Repo) -> Result<(u64, u64)> {
+    const MAX_CONCURRENT: usize = 3;
+
+    let repo_config = repo.config();
+    let verity_state = {
+        let (k, v) = store::REPO_VERITY_CONFIG.split_once('.').unwrap();
+        repo_config
+            .optional_string(k, v)?
+            .map(|v| Tristate::from_str(&v))
+            .transpose()?
+            .unwrap_or_default()
+    };
+
+    let repodir = Dir::reopen_dir(&repo.dfd_borrow())?;
+
+    let mut joinset = tokio::task::JoinSet::new();
+    let mut results = Vec::new();
+
+    for ent in repodir.read_dir("objects")? {
+        while joinset.len() >= MAX_CONCURRENT {
+            results.push(joinset.join_next().await.unwrap()??);
+        }
+        let ent = ent?;
+        if !ent.file_type()?.is_dir() {
+            continue;
+        }
+        let objdir = ent.open_dir()?;
+        joinset.spawn_blocking(move || verity_state_of_objects(&objdir));
+    }
+
+    while let Some(output) = joinset.join_next().await {
+        results.push(output??);
+    }
+    let r = results.into_iter().fold((0, 0), |mut acc, v| {
+        acc.0 += v.0;
+        acc.1 += v.1;
+        acc
+    });
+    Ok(r)
+}
+
+pub(crate) async fn fsck(storage: &Storage) -> Result<FsckResult> {
+    let mut r = FsckResult::default();
+    r.verity = match verity_state_of_all_objects(&storage.repo()).await? {
+        (0, 0) => None,
+        (_, 0) => Some(VerityState::Enabled),
+        (0, _) => Some(VerityState::Disabled),
+        _ => Some(VerityState::Inconsistent),
+    };
+    if matches!(&r.verity, &Some(VerityState::Inconsistent)) {
+        r.errors.push("Inconsistent fsverity state".into());
+    }
+    serde_json::to_writer(std::io::stdout().lock(), &r)?;
+    Ok(r)
+}