wip: Install with fsverity

Signed-off-by: Colin Walters <[email protected]>

.github/workflows/ci.yml

@@ -71,10 +71,17 @@ jobs:
         run: sudo rm -f /bin/skopeo /usr/bin/skopeo
       - name: Free up disk space on runner
         run: sudo ./ci/clean-gha-runner.sh
+      - name: Enable fsverity for /
+        run: sudo tune2fs -O verity $(findmnt -vno SOURCE /)
+      - name: Install utils
+        run: sudo apt -y install fsverity
       - name: Integration tests
         run: |
           set -xeu
+          # Build images to test; TODO investigate doing single container builds
+          # via GHA and pushing to a temporary registry to share among workflows?
           sudo podman build -t localhost/bootc -f hack/Containerfile .
+          sudo podman build -t localhost/bootc-fsverity -f ci/Containerfile.install-fsverity
           export CARGO_INCREMENTAL=0  # because we aren't caching the test runner bits
           cargo build --release -p tests-integration
           df -h /
@@ -83,8 +90,9 @@ jobs:
           df -h /
           # Nondestructive but privileged tests
           sudo bootc-integration-tests host-privileged localhost/bootc
-          # Finally the install-alongside suite
+          # Install tests
           sudo bootc-integration-tests install-alongside localhost/bootc
+          sudo bootc-integration-tests install-fsverity localhost/bootc-fsverity
   docs:
     if: ${{ contains(github.event.pull_request.labels.*.name, 'documentation') }}
     runs-on: ubuntu-latest

ci/Containerfile.install-fsverity

@@ -0,0 +1,10 @@
+# Enable fsverity at install time
+FROM localhost/bootc
+RUN <<EORUN
+set -xeuo pipefail
+mkdir -p /usr/lib/bootc/install
+cat > /usr/lib/bootc/install/30-fsverity.toml <<EOF
+[install]
+fsverity = "enabled"
+EOF
+EORUN

lib/src/cli.rs

@@ -280,6 +280,16 @@ pub(crate) enum ImageOpts {
     Cmd(ImageCmdOpts),
 }
 
+/// Options for consistency checking
+#[derive(Debug, clap::Subcommand, PartialEq, Eq)]
+pub(crate) enum FsckOpts {
+    /// Check the state of fsverity on the ostree objects. Possible output:
+    /// "enabled" => All .file objects have fsverity
+    /// "disabled" => No .file objects have fsverity
+    /// "inconsistent" => Mixed state
+    OstreeVerity
+}
+
 /// Hidden, internal only options
 #[derive(Debug, clap::Subcommand, PartialEq, Eq)]
 pub(crate) enum InternalsOpts {
@@ -293,6 +303,8 @@ pub(crate) enum InternalsOpts {
     FixupEtcFstab,
     /// Should only be used by `make update-generated`
     PrintJsonSchema,
+    /// Perform consistency checking.
+    Fsck(FsckOpts),
     /// Perform cleanup actions
     Cleanup,
     /// Proxy frontend for the `ostree-ext` CLI.

lib/src/install.rs

@@ -33,6 +33,7 @@ use cap_std_ext::cmdext::CapStdExtCommandExt;
 use cap_std_ext::prelude::CapStdExtDirExt;
 use chrono::prelude::*;
 use clap::ValueEnum;
+use config::Tristate;
 use fn_error_context::context;
 use ostree::gio;
 use ostree_ext::container as ostree_container;
@@ -68,6 +69,15 @@ const SELINUXFS: &str = "/sys/fs/selinux";
 const EFIVARFS: &str = "/sys/firmware/efi/efivars";
 pub(crate) const ARCH_USES_EFI: bool = cfg!(any(target_arch = "x86_64", target_arch = "aarch64"));
 
+const DEFAULT_REPO_CONFIG: &[(&str, &str)] = &[
+    // Default to avoiding grub2-mkconfig etc.
+    ("sysroot.bootloader", "none"),
+    // Always flip this one on because we need to support alongside installs
+    // to systems without a separate boot partition.
+    ("sysroot.bootprefix", "true"),
+    ("sysroot.readonly", "true"),
+];
+
 /// Kernel argument used to specify we want the rootfs mounted read-write by default
 const RW_KARG: &str = "rw";
 
@@ -577,6 +587,7 @@ pub(crate) fn print_configuration() -> Result<()> {
 
 #[context("Creating ostree deployment")]
 async fn initialize_ostree_root(state: &State, root_setup: &RootSetup) -> Result<Storage> {
+    let install_config = state.install_config.as_ref();
     let sepolicy = state.load_policy()?;
     let sepolicy = sepolicy.as_ref();
     // Load a fd for the mounted target physical root
@@ -602,14 +613,19 @@ async fn initialize_ostree_root(state: &State, root_setup: &RootSetup) -> Result
         crate::lsm::ensure_dir_labeled(rootfs_dir, "boot", None, 0o755.into(), sepolicy)?;
     }
 
-    for (k, v) in [
-        // Default to avoiding grub2-mkconfig etc.
-        ("sysroot.bootloader", "none"),
-        // Always flip this one on because we need to support alongside installs
-        // to systems without a separate boot partition.
-        ("sysroot.bootprefix", "true"),
-        ("sysroot.readonly", "true"),
-    ] {
+    let fsverity = install_config
+        .and_then(|c| c.fsverity.clone())
+        .unwrap_or_default();
+    let fsverity_ostree_key = "ex-integrity.fsverity";
+    let fsverity_ostree_opt = match fsverity {
+        Tristate::Disabled => None,
+        Tristate::Optional => Some((fsverity_ostree_key, "maybe")),
+        Tristate::Enabled => Some((fsverity_ostree_key, "yes")),
+    };
+    for (k, v) in DEFAULT_REPO_CONFIG
+        .iter()
+        .chain(fsverity_ostree_opt.as_ref())
+    {
         Command::new("ostree")
             .args(["config", "--repo", "ostree/repo", "set", k, v])
             .cwd_dir(rootfs_dir.try_clone()?)

lib/src/install/baseline.rs

@@ -24,6 +24,7 @@ use serde::{Deserialize, Serialize};
 use super::MountSpec;
 use super::RootSetup;
 use super::State;
+use super::Tristate;
 use super::RUN_BOOTC;
 use super::RW_KARG;
 use crate::mount;
@@ -147,13 +148,12 @@ pub(crate) fn install_create_rootfs(
     state: &State,
     opts: InstallBlockDeviceOpts,
 ) -> Result<RootSetup> {
+    let install_config = state.install_config.as_ref();
     let luks_name = "root";
     // Ensure we have a root filesystem upfront
     let root_filesystem = opts
         .filesystem
-        .or(state
-            .install_config
-            .as_ref()
+        .or(install_config
             .and_then(|c| c.filesystem_root())
             .and_then(|r| r.fstype))
         .ok_or_else(|| anyhow::anyhow!("No root filesystem specified"))?;
@@ -192,7 +192,7 @@ pub(crate) fn install_create_rootfs(
     }
 
     // Use the install configuration to find the block setup, if we have one
-    let block_setup = if let Some(config) = state.install_config.as_ref() {
+    let block_setup = if let Some(config) = install_config {
         config.get_block_setup(opts.block_setup.as_ref().copied())?
     } else if opts.filesystem.is_some() {
         // Otherwise, if a filesystem is specified then we default to whatever was
@@ -370,8 +370,18 @@ pub(crate) fn install_create_rootfs(
         None
     };
 
+    let fsverity = install_config
+        .and_then(|c| c.fsverity.clone())
+        .unwrap_or_default();
+    let mkfs_options = match (root_filesystem, fsverity) {
+        (Filesystem::Ext4, Tristate::Enabled | Tristate::Optional) => ["-O", "verity"].as_slice(),
+        _ => [].as_slice(),
+    }
+    .iter()
+    .copied();
+
     // Initialize rootfs
-    let root_uuid = mkfs(&rootdev, root_filesystem, "root", opts.wipe, [])?;
+    let root_uuid = mkfs(&rootdev, root_filesystem, "root", opts.wipe, mkfs_options)?;
     let rootarg = format!("root=UUID={root_uuid}");
     let bootsrc = boot_uuid.as_ref().map(|uuid| format!("UUID={uuid}"));
     let bootarg = bootsrc.as_deref().map(|bootsrc| format!("boot={bootsrc}"));

lib/src/install/config.rs

@@ -41,6 +41,18 @@ pub(crate) struct BasicFilesystems {
     // pub(crate) esp: Option<FilesystemCustomization>,
 }
 
+#[derive(Debug, Clone, Serialize, Deserialize, Default, PartialEq, Eq)]
+#[serde(deny_unknown_fields, rename_all = "kebab-case")]
+pub(crate) enum Tristate {
+    #[default]
+    // The feature is disabled
+    Disabled,
+    // The feature is enabled if supported
+    Optional,
+    // The feature is enabled
+    Enabled,
+}
+
 /// The serialized [install] section
 #[derive(Debug, Clone, Serialize, Deserialize, Default)]
 #[serde(rename = "install", rename_all = "kebab-case", deny_unknown_fields)]
@@ -50,6 +62,8 @@ pub(crate) struct InstallConfiguration {
     /// Enabled block storage configurations
     pub(crate) block: Option<Vec<BlockSetup>>,
     pub(crate) filesystem: Option<BasicFilesystems>,
+    /// How we should use fsverity.
+    pub(crate) fsverity: Option<Tristate>,
     /// Kernel arguments, applied at installation time
     #[serde(skip_serializing_if = "Option::is_none")]
     pub(crate) kargs: Option<Vec<String>>,
@@ -113,6 +127,7 @@ impl Mergeable for InstallConfiguration {
         {
             merge_basic(&mut self.root_fs_type, other.root_fs_type, env);
             merge_basic(&mut self.block, other.block, env);
+            merge_basic(&mut self.fsverity, other.fsverity, env);
             self.filesystem.merge(other.filesystem, env);
             if let Some(other_kargs) = other.kargs {
                 self.kargs
@@ -550,3 +565,29 @@ root-fs-type = "xfs"
         )
     );
 }
+
+#[test]
+/// Test parsing fsverity
+fn test_fsverity() {
+    let env = EnvProperties {
+        sys_arch: "aarch64".to_string(),
+    };
+    let mut c: InstallConfigurationToplevel = toml::from_str(
+        r##"[install]
+root-fs-type = "xfs"
+fsverity = "enabled"
+"##,
+    )
+    .unwrap();
+    let install = c.install.as_ref().unwrap();
+    assert_eq!(install.fsverity.as_ref().unwrap(), &Tristate::Enabled);
+    let o: InstallConfigurationToplevel = toml::from_str(
+        r##"[install]
+fsverity = "optional"
+"##,
+    )
+    .unwrap();
+    c.install.merge(o.install, &env);
+    let install = c.install.as_ref().unwrap();
+    assert_eq!(install.fsverity.as_ref().unwrap(), &Tristate::Optional);
+}

tests-integration/src/install.rs

@@ -133,6 +133,10 @@ pub(crate) fn run_alongside(image: &str, mut testargs: libtest_mimic::Arguments)
                     "grep authorized_keys etc/tmpfiles.d/bootc-root-ssh.conf"
                 )
                 .run()?;
+
+                // Also check that fsverity is disabled
+                assert!(cmd!(sh, "fsverity measure usr/bin/bash").run().is_err()));
+
                 drop(cwd);
                 Ok(())
             },
@@ -170,3 +174,39 @@ pub(crate) fn run_alongside(image: &str, mut testargs: libtest_mimic::Arguments)
 
     libtest_mimic::run(&testargs, tests.into()).exit()
 }
+
+
+#[context("Install tests")]
+pub(crate) fn run_fsverity(image: &str, mut testargs: libtest_mimic::Arguments) -> Result<()> {
+    // Force all of these tests to be serial because they mutate global state
+    testargs.test_threads = Some(1);
+    // Just leak the image name so we get a static reference as required by the test framework
+    let image: &'static str = String::from(image).leak();
+    // Handy defaults
+
+    let target_args = &["-v", "/:/target"];
+    // We always need this as we assume we're operating on a local image
+    let generic_inst_args = ["--skip-fetch-check"];
+
+    let tests = [
+        Trial::test("Install and verify fsverity", move || {
+            let sh = &xshell::Shell::new()?;
+            reset_root(sh, image)?;
+            cmd!(sh, "sudo {BASE_ARGS...} {target_args...} {image} bootc install to-existing-root --acknowledge-destructive {generic_inst_args...}").run()?;
+            generic_post_install_verification()?;
+            let objects00 = Dir::open_ambient_dir("/ostree/repo/objects/00", cap_std::ambient_authority())?;
+            for ent in objects00.entries()? {
+                let ent = ent?;
+                let name = ent.file_name();
+                let name = name.to_str().unwrap();
+                if !name.ends_with(".file") {
+                    continue
+                }
+                cmd!(sh, "fsverity measure {name}").
+            }
+            Ok(())
+        }),
+    ];
+
+    libtest_mimic::run(&testargs, tests.into()).exit()
+}

tests-integration/src/tests-integration.rs

@@ -48,6 +48,7 @@ fn main() {
     let opt = Opt::parse();
     let r = match opt {
         Opt::InstallAlongside { image, testargs } => install::run_alongside(&image, testargs),
+        Opt::InstallFsverity { image, testargs } => install::run_fsverity(&image, testargs),
         Opt::HostPrivileged { image, testargs } => hostpriv::run_hostpriv(&image, testargs),
         Opt::Container { testargs } => container::run(testargs),
         Opt::RunVM(opts) => runvm::run(opts),