Wire up bits for CPUID customization

This is a first cut at making CPUID information customizable, rather
than using the in-kernel vmm logic which applies limited transformations
to the data exposed by the host CPU.  Basic plumbing in propolis-lib is
provided, which is wired into propolis-standalone.  The same
configuration syntax is defined for propolis-server, but it is going
unused for now until some internal structure is hammered out.

Cargo.toml

@@ -43,6 +43,7 @@ panic = "abort"
 # Internal crates
 bhyve_api = { path = "crates/bhyve-api" }
 bhyve_api_sys = { path = "crates/bhyve-api/sys" }
+cpuid_profile_config = { path = "crates/cpuid-profile-config" }
 dladm = { path = "crates/dladm" }
 propolis-server-config = { path = "crates/propolis-server-config" }
 propolis-standalone-config = { path = "crates/propolis-standalone-config" }

bin/propolis-server/src/lib/spec.rs

@@ -603,13 +603,11 @@ impl ServerSpecBuilder {
 
 #[cfg(test)]
 mod test {
-    use std::{collections::BTreeMap, path::PathBuf};
-
     use crucible_client_types::VolumeConstructionRequest;
     use propolis_client::handmade::api::Slot;
     use uuid::Uuid;
 
-    use crate::config::{self, Config};
+    use crate::config::Config;
 
     use super::*;
 
@@ -625,13 +623,7 @@ mod test {
                 memory: 512,
                 vcpus: 4,
             },
-            &Config::new(
-                PathBuf::from_str("").unwrap(),
-                config::Chipset::default(),
-                BTreeMap::new(),
-                BTreeMap::new(),
-                Vec::new(),
-            ),
+            &Config::default(),
         )
     }
 

bin/propolis-standalone/README.md

@@ -101,9 +101,10 @@ fi
 After you've got the bootrom, an ISO, a VNIC, and a configuration file that
 points to them, you're ready to create and run your VM. To do so, make sure
 you've done the following:
-- [build propolis](#Building)
-- run the [propolis-server](#propolis-server)
-- create your VM, run it, and hop on the serial console using [propolis-cli](#propolis-cli)
+- build `propolis-standalone`
+- start `propolis-standalone`, passing it a valid config
+- it will wait to start the VM until you connect to the serial console socket
+  (with something like [sercons](https://github.com/jclulow/vmware-sercons))
 - login to the VM as root (no password)
 - optionally, run `setup-alpine` to configure the VM (including setting a root
   password)
@@ -190,3 +191,63 @@ generation = 1
 # read_only = false
 # === END OPTIONAL OPTIONS ===
 ```
+## Configuring `cpuid`
+
+Rather than using the built-in `cpuid` data masking offered by the bhyve kernel
+VMM, propolis-standalone can load a set of leaf data to be used by the instance.
+An example of such configuration data is as follows:
+
+```toml
+[main]
+# ... other main config bits
+cpuid_profile = "NAME"
+
+[cpuid.NAME]
+vendor = "amd"
+"0" = [0x10, 0x68747541, 0x444d4163, 0x69746e65]
+"1" = [0x830f10, 0x10800, 0xf6d83203, 0x178bfbff]
+"5" = [0x0, 0x0, 0x0, 0x0]
+"6" = [0x4, 0x0, 0x0, 0x0]
+"7" = [0x0, 0x0, 0x0, 0x0]
+"7-0" = [0x0, 0x201401a9, 0x0, 0x0]
+"d" = [0x0, 0x0, 0x0, 0x0]
+"d-0" = [0x7, 0x340, 0x340, 0x0]
+"d-1" = [0x1, 0x0, 0x0, 0x0]
+"d-2" = [0x100, 0x240, 0x0, 0x0]
+"80000000" = [0x80000020, 0x68747541, 0x444d4163, 0x69746e65]
+"80000001" = [0x830f10, 0x40000000, 0x444031fb, 0x25d3fbff]
+"80000002" = [0x20444d41, 0x43595045, 0x38323720, 0x36312032]
+"80000003" = [0x726f432d, 0x72502065, 0x7365636f, 0x20726f73]
+"80000004" = [0x20202020, 0x20202020, 0x20202020, 0x202020]
+"80000005" = [0xff40ff40, 0xff40ff40, 0x20080140, 0x20080140]
+"80000006" = [0x48006400, 0x68006400, 0x2006140, 0x2009140]
+"80000007" = [0x0, 0x0, 0x0, 0x100]
+"80000008" = [0x3030, 0x7, 0x0, 0x10000]
+"8000000a" = [0x1, 0x8000, 0x0, 0x13bcff]
+"80000019" = [0xf040f040, 0x0, 0x0, 0x0]
+"8000001a" = [0x6, 0x0, 0x0, 0x0]
+"8000001b" = [0x3ff, 0x0, 0x0, 0x0]
+"8000001d" = [0x0, 0x0, 0x0, 0x0]
+"8000001d-0" = [0x121, 0x1c0003f, 0x3f, 0x0]
+"8000001d-1" = [0x122, 0x1c0003f, 0x3f, 0x0]
+"8000001d-2" = [0x143, 0x1c0003f, 0x3ff, 0x2]
+"8000001d-3" = [0x163, 0x3c0003f, 0x3fff, 0x1]
+"8000001f" = [0x1000f, 0x16f, 0x1fd, 0x1]
+```
+
+If `cpuid_profile` is specified under the `main` section, a corresponding
+`cpuid` section with a matching name is expected to be defined elsewhere in the
+file.  The `vendor` field under that section controls fallback behavior when a
+vCPU queries a non-existent leaf, and other CPU-specific behavior.  After that,
+the leafs and their register data are listed.  Leafs which require an `ecx`
+match (with `eax` as the function, and `ecx` as the index) are specified with a
+hyphen separating the function and index.  Leafs without an index (just a single
+hex number) will match only against `eax`, and at a lower priority than the
+function/index leafs which match `eax` and `ecx`.  The data for leafs is
+expected to be a 4-item array of 32-bit integers corresponding to `eax`, `ebx`,
+`ecx`, and `edx`, in that order.
+
+Certain fields in `cpuid` data depend on aspects specific to the host (such as
+vCPU count) or the vCPU they are associated with (such as APIC ID).  Propolis
+will "specialize" the data provided in the `cpuid` profile with logic appropriate
+for the specific leafs involved.

bin/propolis-standalone/src/config.rs

@@ -11,11 +11,12 @@ use anyhow::Context;
 use serde::Deserialize;
 
 use propolis::block;
+use propolis::cpuid;
 use propolis::hw::pci::Bdf;
 use propolis::inventory::ChildRegister;
 
-use propolis_standalone_config::Device;
 pub use propolis_standalone_config::{Config, SnapshotTag};
+use propolis_standalone_config::{CpuVendor, CpuidEntry, Device};
 
 #[derive(Deserialize)]
 struct FileConfig {
@@ -119,6 +120,34 @@ pub fn parse_bdf(v: &str) -> Option<Bdf> {
     }
 }
 
+pub fn parse_cpuid(config: &Config) -> anyhow::Result<Option<cpuid::Set>> {
+    if let Some(profile) = config.cpuid_profile() {
+        let vendor = match profile.vendor {
+            CpuVendor::Amd => cpuid::VendorKind::Amd,
+            CpuVendor::Intel => cpuid::VendorKind::Intel,
+        };
+        let mut set = cpuid::Set::new(vendor);
+        let entries: Vec<CpuidEntry> = profile.try_into()?;
+        for entry in entries {
+            let conflict = set.insert(
+                cpuid::Ident(entry.func, entry.idx),
+                cpuid::Entry::from(entry.values),
+            );
+
+            if conflict.is_some() {
+                anyhow::bail!(
+                    "conflicing entry at func:{:#?} idx:{:#?}",
+                    entry.func,
+                    entry.idx
+                )
+            }
+        }
+        Ok(Some(set))
+    } else {
+        Ok(None)
+    }
+}
+
 #[cfg(feature = "crucible")]
 fn create_crucible_backend(
     be: &propolis_standalone_config::BlockDevice,

bin/propolis-standalone/src/main.rs

@@ -923,7 +923,26 @@ fn setup_instance(
     inv.register(&fwcfg_dev)?;
     inv.register(&ramfb)?;
 
+    let cpuid_profile = config::parse_cpuid(&config)?;
+
     for vcpu in machine.vcpus.iter() {
+        let vcpu_profile = if let Some(profile) = cpuid_profile.as_ref() {
+            propolis::cpuid::Specializer::new()
+                .with_vcpu_count(
+                    std::num::NonZeroU8::new(config.main.cpus).unwrap(),
+                    true,
+                )
+                .with_vcpuid(vcpu.id)
+                .with_cache_topo()
+                .clear_cpu_topo(cpuid::TopoKind::all())
+                .execute(profile.clone())
+                .context("failed to specialize cpuid profile")?
+        } else {
+            // An empty set will instruct the kernel to use the legacy
+            // fallback behavior
+            propolis::cpuid::Set::new_host()
+        };
+        vcpu.set_cpuid(vcpu_profile)?;
         vcpu.set_default_capabs()?;
     }
     drop(guard);

bin/propolis-utils/README.md

@@ -7,7 +7,7 @@ development activities, but are otherwise not meant for general consumption.
 
 - `savex`:  Extract data fields from a `propolis-standalone` instance which was
   halted and saved with the `-s` flag.
-- `cpuid-gen`: Generated CPUID profile using the legacy emulated output from the
+- `cpuid-gen`: Generated `cpuid` profile using the legacy emulated output from the
   local host CPU, as filtered by the kernel VMM logic.
 - `rsrvrctl`: Manipulate the kernel VMM memory reservoir in the same manner
   offered by the utility shipped by the OS

bin/propolis-utils/src/bin/cpuid-gen.rs

@@ -96,7 +96,7 @@ impl PartialOrd for CpuidKey {
     }
 }
 
-/// Query CPUID through bhyve-defined masks
+/// Query `cpuid` through bhyve-defined masks
 fn query_cpuid(vm: &VmmFd, eax: u32, ecx: u32) -> anyhow::Result<Cpuid> {
     let mut data = bhyve_api::vm_legacy_cpuid {
         vlc_eax: eax,
@@ -107,7 +107,7 @@ fn query_cpuid(vm: &VmmFd, eax: u32, ecx: u32) -> anyhow::Result<Cpuid> {
     Ok(Cpuid::from(&data))
 }
 
-/// Query CPUID directly from host CPU
+/// Query `cpuid` directly from host CPU
 fn query_raw_cpuid(eax: u32, ecx: u32) -> Cpuid {
     let mut res = Cpuid::default();
 

crates/bhyve-api/src/lib.rs

@@ -578,7 +578,7 @@ pub enum ApiVersion {
     /// Made hlt-on-exit a required CPU feature, and enabled by default in vmm
     V6 = 6,
 
-    /// Adds ability to control CPUID results for guest vCPUs
+    /// Adds ability to control `cpuid` results for guest vCPUs
     V5 = 5,
 }
 impl ApiVersion {

crates/bhyve-api/sys/src/structs.rs

@@ -392,16 +392,57 @@ impl Default for vm_data_xfer {
 pub const VCE_FLAG_MATCH_INDEX: u32 = 1 << 0;
 
 #[repr(C)]
-#[derive(Copy, Clone)]
-struct vcpu_cpuid_entry {
-    vce_function: u32,
-    vce_index: u32,
-    vce_flags: u32,
-    vce_eax: u32,
-    vce_ebx: u32,
-    vce_ecx: u32,
-    vce_edx: u32,
-    _pad: u32,
+#[derive(Copy, Clone, Default)]
+pub struct vcpu_cpuid_entry {
+    pub vce_function: u32,
+    pub vce_index: u32,
+    pub vce_flags: u32,
+    pub vce_eax: u32,
+    pub vce_ebx: u32,
+    pub vce_ecx: u32,
+    pub vce_edx: u32,
+    pub _pad: u32,
+}
+impl vcpu_cpuid_entry {
+    fn match_idx(&self) -> bool {
+        self.vce_flags & VCE_FLAG_MATCH_INDEX != 0
+    }
+    /// Order entries for proper cpuid evaluation by the kernel VMM.
+    ///
+    /// Bhyve expects that cpuid entries are sorted by function, and then index,
+    /// from least to greatest.  Entries which must match on index should come
+    /// before (less-than) those that do not, so the former can take precedence
+    /// in matching.
+    ///
+    /// This function is provided so that a list of entries can be easily sorted
+    /// prior to loading them into the kernel VMM.
+    ///
+    /// ```
+    /// let mut entries: Vec<vcpu_cpuid_entry> = vec![
+    ///     // entries loaded here
+    /// ];
+    /// entries.sort_by(vcpu_cpuid_entry::eval_sort);
+    /// let config = vm_vcpu_cpuid_config {
+    ///     vvcc_cpuid: 0,
+    ///     vvcc_flags: 0,
+    ///     vvcc_nent: entries.len(),
+    ///     vvcc_entries: &mut entries,
+    /// };
+    /// // perform ioctl(VM_SET_CPUID, &config) ...
+    /// ```
+    pub fn eval_sort(a: &Self, b: &Self) -> std::cmp::Ordering {
+        use std::cmp::Ordering;
+
+        match a.vce_function.cmp(&b.vce_function) {
+            Ordering::Equal => match (a.match_idx(), b.match_idx()) {
+                (true, false) => Ordering::Less,
+                (false, true) => Ordering::Greater,
+                (true, true) | (false, false) => a.vce_index.cmp(&b.vce_index),
+            },
+
+            ord => ord,
+        }
+    }
 }
 
 /// Use legacy hard-coded cpuid masking tables applied to the host CPU
@@ -421,6 +462,17 @@ pub struct vm_vcpu_cpuid_config {
     pub _pad: u32,
     pub vvcc_entries: *mut c_void,
 }
+impl Default for vm_vcpu_cpuid_config {
+    fn default() -> Self {
+        Self {
+            vvcc_vcpuid: 0,
+            vvcc_flags: 0,
+            vvcc_nent: 0,
+            _pad: 0,
+            vvcc_entries: std::ptr::null_mut(),
+        }
+    }
+}
 
 #[repr(C)]
 #[derive(Copy, Clone, Default)]

crates/cpuid-profile-config/Cargo.toml

@@ -0,0 +1,16 @@
+[package]
+name = "cpuid_profile_config"
+version = "0.0.0"
+license = "MPL-2.0"
+edition = "2021"
+
+[lib]
+test = false
+doctest = false
+
+[dependencies]
+serde.workspace = true
+serde_derive.workspace = true
+toml.workspace = true
+propolis.workspace = true
+thiserror.workspace = true