diff --git a/.cargo/config.toml b/.cargo/config.toml new file mode 100644 index 0000000..ff7f758 --- /dev/null +++ b/.cargo/config.toml @@ -0,0 +1,2 @@ +[target.aarch64-unknown-linux-gnu] +linker = "aarch64-linux-gnu-gcc" \ No newline at end of file diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml new file mode 100644 index 0000000..c5111e7 --- /dev/null +++ b/.github/workflows/rust.yml @@ -0,0 +1,30 @@ +name: Rust + +on: + push: + branches: [ "main" ] + pull_request: + branches: [ "main" ] + +env: + CARGO_TERM_COLOR: always + +jobs: + build: + + runs-on: ubuntu-latest + strategy: + matrix: + toolchain: + - stable + + steps: + - uses: actions/checkout@v3 + - run: rustup update ${{ matrix.toolchain }} && rustup default ${{ matrix.toolchain }} + - run: rustup component add rustfmt + - name: Build + run: cargo build --verbose + - name: Run tests + run: cargo test --verbose + - name: Run rustfmt + run: cargo fmt -- --check diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..ea8c4bf --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +/target diff --git a/.vscode/settings.json b/.vscode/settings.json new file mode 100644 index 0000000..2430cd6 --- /dev/null +++ b/.vscode/settings.json @@ -0,0 +1,6 @@ +{ + "rust-analyzer.linkedProjects": [ + "./src/vmm/Cargo.toml" + ], + "rust-analyzer.showUnlinkedFileNotification": false +} \ No newline at end of file diff --git a/CONTRIBUTORS b/CONTRIBUTORS new file mode 100644 index 0000000..2853dd7 --- /dev/null +++ b/CONTRIBUTORS @@ -0,0 +1,4 @@ +# This is a list of people who contributed code to the bao hypervisor by +# chronological order of first contribution. + +joaopeixoto13 \ No newline at end of file diff --git a/Cargo.lock b/Cargo.lock new file mode 100644 index 0000000..5dce4fa --- /dev/null +++ b/Cargo.lock @@ -0,0 +1,530 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 3 + +[[package]] +name = "aarch64-cpu" +version = "9.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac42a04a61c19fc8196dd728022a784baecc5d63d7e256c01ad1b3fbfab26287" +dependencies = [ + "tock-registers", +] + +[[package]] +name = "api" +version = "0.1.0" +dependencies = [ + "aarch64-cpu", + "clap", + "event-manager", + "libc", + "once_cell", + "serde", + "serde_yaml", + "thiserror", + "vhost-user-frontend", + "vm-memory", + "vmm-sys-util", +] + +[[package]] +name = "arc-swap" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bddcadddf5e9015d310179a59bb28c4d4b9920ad0f11e8e14dbadf654890c9a6" + +[[package]] +name = "atty" +version = "0.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8" +dependencies = [ + "hermit-abi", + "libc", + "winapi", +] + +[[package]] +name = "autocfg" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" + +[[package]] +name = "bao-virtio" +version = "0.1.0" +dependencies = [ + "api", + "field-offset", + "lazy_static", + "libc", + "log", + "seccompiler", + "virtio", + "vmm", +] + +[[package]] +name = "bitflags" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" + +[[package]] +name = "bitflags" +version = "2.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed570934406eb16438a4e976b1b4500774099c13b8cb96eec99f620f05090ddf" + +[[package]] +name = "clap" +version = "3.2.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ea181bf566f71cb9a5d17a59e1871af638180a18fb0035c92ae62b705207123" +dependencies = [ + "atty", + "bitflags 1.3.2", + "clap_lex", + "indexmap", + "strsim", + "termcolor", + "textwrap", +] + +[[package]] +name = "clap_lex" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2850f2f5a82cbf437dd5af4d49848fbdfc27c157c3d010345776f952765261c5" +dependencies = [ + "os_str_bytes", +] + +[[package]] +name = "epoll" +version = "4.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "74351c3392ea1ff6cd2628e0042d268ac2371cb613252ff383b6dfa50d22fa79" +dependencies = [ + "bitflags 2.4.2", + "libc", +] + +[[package]] +name = "event-manager" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "90b16fe5161a1160c9c7cece9f7504f2412ef5e2c0643d1e322eccf37692a42b" +dependencies = [ + "libc", + "vmm-sys-util", +] + +[[package]] +name = "field-offset" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38e2275cc4e4fc009b0669731a1e5ab7ebf11f469eaede2bab9309a5b4d6057f" +dependencies = [ + "memoffset", + "rustc_version", +] + +[[package]] +name = "hashbrown" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" + +[[package]] +name = "hermit-abi" +version = "0.1.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "62b467343b94ba476dcb2500d242dadbb39557df889310ac77c5d99100aaac33" +dependencies = [ + "libc", +] + +[[package]] +name = "indexmap" +version = "1.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bd070e393353796e801d209ad339e89596eb4c8d430d18ede6a1cced8fafbd99" +dependencies = [ + "autocfg", + "hashbrown", +] + +[[package]] +name = "lazy_static" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" + +[[package]] +name = "libc" +version = "0.2.153" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c198f91728a82281a64e1f4f9eeb25d82cb32a5de251c6bd1b5154d63a8e7bd" + +[[package]] +name = "linked-hash-map" +version = "0.5.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0717cef1bc8b636c6e1c1bbdefc09e6322da8a9321966e8928ef80d20f7f770f" + +[[package]] +name = "log" +version = "0.4.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5e6163cb8c49088c2c36f57875e58ccd8c87c7427f7fbd50ea6710b2f3f2e8f" + +[[package]] +name = "memoffset" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a634b1c61a95585bd15607c6ab0c4e5b226e695ff2800ba0cdccddf208c406c" +dependencies = [ + "autocfg", +] + +[[package]] +name = "once_cell" +version = "1.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92" + +[[package]] +name = "os_str_bytes" +version = "6.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2355d85b9a3786f481747ced0e0ff2ba35213a1f9bd406ed906554d7af805a1" + +[[package]] +name = "proc-macro2" +version = "1.0.78" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2422ad645d89c99f8f3e6b88a9fdeca7fabeac836b1002371c4367c8f984aae" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "291ec9ab5efd934aaf503a6466c5d5251535d108ee747472c3977cc5acc868ef" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "rustc_version" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfa0f585226d2e68097d4f95d113b15b83a82e819ab25717ec0590d9584ef366" +dependencies = [ + "semver", +] + +[[package]] +name = "ryu" +version = "1.0.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e86697c916019a8588c99b5fac3cead74ec0b4b819707a682fd4d23fa0ce1ba1" + +[[package]] +name = "seccompiler" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e01d1292a1131b22ccea49f30bd106f1238b5ddeec1a98d39268dcc31d540e68" +dependencies = [ + "libc", +] + +[[package]] +name = "semver" +version = "1.0.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92d43fe69e652f3df9bdc2b85b2854a0825b86e4fb76bc44d945137d053639ca" + +[[package]] +name = "serde" +version = "1.0.196" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "870026e60fa08c69f064aa766c10f10b1d62db9ccd4d0abb206472bee0ce3b32" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.196" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33c85360c95e7d137454dc81d9a4ed2b8efd8fbe19cee57357b32b9771fccb67" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_yaml" +version = "0.8.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "578a7433b776b56a35785ed5ce9a7e777ac0598aac5a6dd1b4b18a307c7fc71b" +dependencies = [ + "indexmap", + "ryu", + "serde", + "yaml-rust", +] + +[[package]] +name = "strsim" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623" + +[[package]] +name = "syn" +version = "2.0.49" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "915aea9e586f80826ee59f8453c1101f9d1c4b3964cd2460185ee8e299ada496" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "termcolor" +version = "1.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06794f8f6c5c898b3275aebefa6b8a1cb24cd2c6c79397ab15774837a0bc5755" +dependencies = [ + "winapi-util", +] + +[[package]] +name = "textwrap" +version = "0.16.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23d434d3f8967a09480fb04132ebe0a3e088c173e6d0ee7897abbdf4eab0f8b9" + +[[package]] +name = "thiserror" +version = "1.0.57" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e45bcbe8ed29775f228095caf2cd67af7a4ccf756ebff23a306bf3e8b47b24b" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "1.0.57" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a953cb265bef375dae3de6663da4d3804eee9682ea80d8e2542529b73c531c81" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tock-registers" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "696941a0aee7e276a165a978b37918fd5d22c55c3d6bda197813070ca9c0f21c" + +[[package]] +name = "unicode-ident" +version = "1.0.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b" + +[[package]] +name = "vhost" +version = "0.10.0" +source = "git+https://github.com/joaopeixoto13/vhost?branch=vhost-user-frontend#cf0c49322d8bff5f55591421298c9cb6699279a1" +dependencies = [ + "bitflags 2.4.2", + "libc", + "vm-memory", + "vmm-sys-util", +] + +[[package]] +name = "vhost-user-frontend" +version = "0.1.0" +source = "git+https://github.com/joaopeixoto13/vhost?branch=vhost-user-frontend#cf0c49322d8bff5f55591421298c9cb6699279a1" +dependencies = [ + "epoll", + "libc", + "log", + "seccompiler", + "thiserror", + "vhost", + "virtio-bindings 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)", + "virtio-queue", + "vm-memory", + "vmm-sys-util", +] + +[[package]] +name = "virtio" +version = "0.1.0" +dependencies = [ + "api", + "event-manager", + "libc", + "log", + "seccompiler", + "vhost", + "vhost-user-frontend", + "virtio-bindings 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)", + "virtio-blk", + "virtio-device", + "virtio-queue", + "virtio-vsock", + "vm-device", + "vm-memory", + "vmm-sys-util", +] + +[[package]] +name = "virtio-bindings" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "878bcb1b2812a10c30d53b0ed054999de3d98f25ece91fc173973f9c57aaae86" + +[[package]] +name = "virtio-bindings" +version = "0.2.2" +source = "git+https://github.com/joaopeixoto13/vm-virtio?branch=bao-hypervisor#8e7c29ce0a606c4beeef2e09356ca58f41a72e4b" + +[[package]] +name = "virtio-blk" +version = "0.1.0" +source = "git+https://github.com/joaopeixoto13/vm-virtio?branch=bao-hypervisor#8e7c29ce0a606c4beeef2e09356ca58f41a72e4b" +dependencies = [ + "log", + "virtio-bindings 0.2.2 (git+https://github.com/joaopeixoto13/vm-virtio?branch=bao-hypervisor)", + "virtio-device", + "virtio-queue", + "vm-memory", + "vmm-sys-util", +] + +[[package]] +name = "virtio-device" +version = "0.1.0" +source = "git+https://github.com/joaopeixoto13/vm-virtio?branch=bao-hypervisor#8e7c29ce0a606c4beeef2e09356ca58f41a72e4b" +dependencies = [ + "log", + "virtio-bindings 0.2.2 (git+https://github.com/joaopeixoto13/vm-virtio?branch=bao-hypervisor)", + "virtio-queue", + "vm-memory", +] + +[[package]] +name = "virtio-queue" +version = "0.11.0" +source = "git+https://github.com/joaopeixoto13/vm-virtio?branch=bao-hypervisor#c608baf80cb8b6aa73257b58346b7e7f27d7e489" +dependencies = [ + "log", + "virtio-bindings 0.2.2 (git+https://github.com/joaopeixoto13/vm-virtio?branch=bao-hypervisor)", + "vm-memory", + "vmm-sys-util", +] + +[[package]] +name = "virtio-vsock" +version = "0.5.0" +source = "git+https://github.com/joaopeixoto13/vm-virtio?branch=bao-hypervisor#8e7c29ce0a606c4beeef2e09356ca58f41a72e4b" +dependencies = [ + "virtio-bindings 0.2.2 (git+https://github.com/joaopeixoto13/vm-virtio?branch=bao-hypervisor)", + "virtio-queue", + "vm-memory", +] + +[[package]] +name = "vm-device" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "599adbdaddea4947ca23c085d2b8e47f3499ccda35438424526f3853748a8eb6" + +[[package]] +name = "vm-memory" +version = "0.14.0" +source = "git+https://github.com/joaopeixoto13/vm-memory?branch=bao-hypervisor#5e5e60591e8d904f5d4966980d442c04daa14599" +dependencies = [ + "arc-swap", + "libc", + "thiserror", + "winapi", +] + +[[package]] +name = "vmm" +version = "0.1.0" +dependencies = [ + "api", + "event-manager", + "libc", + "virtio", + "vm-device", + "vmm-sys-util", +] + +[[package]] +name = "vmm-sys-util" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d1435039746e20da4f8d507a72ee1b916f7b4b05af7a91c093d2c6561934ede" +dependencies = [ + "bitflags 1.3.2", + "libc", +] + +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] +name = "winapi-util" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f29e6f9198ba0d26b4c9f07dbe6f9ed633e1f3d5b8b414090084349e46a52596" +dependencies = [ + "winapi", +] + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + +[[package]] +name = "yaml-rust" +version = "0.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56c1936c4cc7a1c9ab21a1ebb602eb942ba868cbd44a99cb7cdc5892335e1c85" +dependencies = [ + "linked-hash-map", +] diff --git a/Cargo.toml b/Cargo.toml new file mode 100644 index 0000000..e92c50b --- /dev/null +++ b/Cargo.toml @@ -0,0 +1,37 @@ +[package] +name = "bao-virtio" +version = "0.1.0" +edition = "2021" +authors = ["João Peixoto "] +keywords = ["bao", "virtio", "virtualization", "security"] +description = "Bao VirtIO" +license = "Apache-2.0" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[workspace] +members = [ + "src/api", + "src/vmm", + "src/virtio", +] + +[dependencies] +field-offset = "0.3.4" +lazy_static = "1.4.0" +libc = ">=0.2.95" +log = "0.4.17" +seccompiler = "0.2.0" +virtio = { path = "src/virtio" } +api = { path = "src/api" } +vmm = { path = "src/vmm" } + +# The [patch] section is used to override dependencies with custom implementations + +[patch.crates-io] +vm-memory = { git = "https://github.com/joaopeixoto13/vm-memory", branch = "bao-hypervisor" } +vhost = { git = "https://github.com/joaopeixoto13/vhost", branch = "vhost-user-frontend" } +virtio-queue = { git = "https://github.com/joaopeixoto13/vm-virtio", branch = "bao-hypervisor" } +virtio-device = { git = "https://github.com/joaopeixoto13/vm-virtio", branch = "bao-hypervisor" } +virtio-blk = { git = "https://github.com/joaopeixoto13/vm-virtio", branch = "bao-hypervisor" } +virtio-vsock = { git = "https://github.com/joaopeixoto13/vm-virtio", branch = "bao-hypervisor" } \ No newline at end of file diff --git a/README.md b/README.md new file mode 100644 index 0000000..0b816b2 --- /dev/null +++ b/README.md @@ -0,0 +1,43 @@ +# Bao Hypervisor VirtIO Device Model + +## Overview + +This repository offers a user space device model, implemented in the Rust programming +language, enabling integration of various VirtIO device types within the [Bao hypervisor](https://github.com/bao-project/bao-hypervisor) +across diverse Virtual Machines (VMs), ranging from common VirtIO devices to Vhost and +Vhost-user backend devices. + +## Getting Started + +To begin utilizing VirtIO device support in Bao Hypervisor, follow these steps: + +1. Clone this repository to your local environment. + +``` +git clone git@github.com:joaopeixoto13/bao-virtio.git +``` + +2. Build the source code (e.g. Aarch64): + +``` +cargo build --target=aarch64-unknown-linux-gnu --release +``` + +## Supported Devices + +The full list of supported (and work in progress) devices is presented below: + +| | DEVICE | DATAPLANE | SUPPORTED | +| ------------------- | ----------------- | ------- | --- | +| Virtio-Block | [Block](src/virtio/src/block/README.md) | VirtIO | [x](src/virtio/src/block/virtio/README.md) | +| Vhost-User-Fs | [Virtual File System](src/virtio/src/fs/README.md) | Vhost-user | [x](src/virtio/src/fs/vhost_user/README.md) | +| Virtio-Net | [Net](src/virtio/src/net/README.md) | VirtIO | [x](src/virtio/src/net/virtio/README.md) | +| Vhost-Net | [Net](src/virtio/src/net/README.md) | Vhost | - | +| Virtio-Vsock | [Vsock](src/virtio/src/vsock/README.md) | VirtIO | - | +| Vhost-Vsock | [Vsock](src/virtio/src/vsock/README.md) | Vhost | [x](src/virtio/src/vsock/vhost/README.md) | +| Vhost-User-Vsock | [Vsock](src/virtio/src/vsock/README.md) | Vhost-user | - | + +## Contributing +Contributions to enhance the functionality and features of Bao Hypervisor VirtIO Device +Support are welcome. If you have suggestions, bug fixes, or new features to propose, +feel free to open an issue or submit a pull request. \ No newline at end of file diff --git a/src/api/.gitignore b/src/api/.gitignore new file mode 100644 index 0000000..ea8c4bf --- /dev/null +++ b/src/api/.gitignore @@ -0,0 +1 @@ +/target diff --git a/src/api/Cargo.toml b/src/api/Cargo.toml new file mode 100644 index 0000000..4536a7d --- /dev/null +++ b/src/api/Cargo.toml @@ -0,0 +1,23 @@ +[package] +name = "api" +version = "0.1.0" +edition = "2021" +authors = ["João Peixoto "] +keywords = ["bao", "api", "parser"] +description = "Bao API" +license = "Apache-2.0" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +clap = "3.0" +serde = { version = "1.0", features = ["derive"] } +serde_yaml = "0.8" +thiserror = "1.0" +libc = ">=0.2.95" +vmm-sys-util = "0.12.1" +vhost-user-frontend = { git = "https://github.com/joaopeixoto13/vhost", branch = "vhost-user-frontend" } +event-manager = { version = "0.4.0", features = ["remote_endpoint"] } +aarch64-cpu = "9.4.0" +vm-memory = { version = "0.14.0", features = ["backend-mmap", "backend-atomic", "backend-bitmap"] } +once_cell = "1.19.0" \ No newline at end of file diff --git a/src/api/src/cli.rs b/src/api/src/cli.rs new file mode 100644 index 0000000..d5b2b75 --- /dev/null +++ b/src/api/src/cli.rs @@ -0,0 +1,126 @@ +// Copyright (c) Bao Project and Contributors. All rights reserved. +// João Peixoto +// +// SPDX-License-Identifier: Apache-2.0 + +//! Bao CLI. + +use super::types::VMMConfig; +use clap::{App, Arg, Error}; +use std::fs::File; +use std::io::Read; + +/// Command line interface. +pub struct Cli; + +impl Cli { + /// Creates a new `Cli` object. + /// + /// # Returns + /// + /// A `Cli` object. + pub fn new() -> Self { + Cli + } + + /// Launches the command line interface. + /// + /// # Examples + /// + /// $ bao-vmm --config /path/to/your/config.yaml + /// + /// or (short version) + /// + /// $ bao-vmm -c /path/to/your/config.yaml + /// + /// # Returns + /// + /// * `Result` - A VMMConfig struct containing the parsed configuration. + pub fn launch(&self) -> Result { + let vmm_config = match self.parse() { + Ok(config) => config, + Err(e) => { + return Err(Error::with_description( + e.to_string(), + clap::ErrorKind::InvalidValue, + )); + } + }; + Ok(vmm_config) + } + + /// Launches the command line interface with a config file. + /// + /// # Arguments + /// + /// * `file_path` - A reference to a string containing the path to the YAML file. + /// + /// # Returns + /// + /// * `Result` - A VMMConfig struct containing the parsed configuration. + pub fn launch_with_file(&self, file_path: &str) -> Result { + let vmm_config = match self.parse_yaml_config_file(file_path) { + Ok(config) => config, + Err(e) => { + return Err(Error::with_description( + e.to_string(), + clap::ErrorKind::InvalidValue, + )); + } + }; + Ok(vmm_config) + } + + /// Parses the VMM arguments. + /// + /// # Returns + /// + /// * `Result>` - A ConfigFrontends struct containing the parsed configuration. + fn parse(&self) -> Result> { + // Get the environment command line arguments + let matches = App::new("Bao Vhost Frontend") + .arg( + Arg::with_name("config") + .short('c') + .long("config") + .value_name("FILE") + .help("Sets a custom config file") + .takes_value(true) + .required(true), + ) + .get_matches(); + + // Extract the config file path + let config_file = matches.value_of("config").unwrap(); + + // Parse the YAML file + let frontends = self.parse_yaml_config_file(config_file)?; + + // Return the configuration + Ok(frontends) + } + + /// Parses the YAML configuration file. + /// + /// # Arguments + /// + /// * `file_path` - A reference to a string containing the path to the YAML file. + /// + /// # Returns + /// + /// * `Result>` - A VMMConfig struct containing the parsed configuration. + fn parse_yaml_config_file( + &self, + file_path: &str, + ) -> Result> { + // Open the YAML file + let mut file = File::open(file_path).unwrap(); + // Read the YAML file + let mut yaml_content = String::new(); + file.read_to_string(&mut yaml_content).unwrap(); + // Parse the YAML file + let vmm_config: VMMConfig = serde_yaml::from_str(&yaml_content).unwrap(); + // Return the configuration + Ok(vmm_config) + } +} diff --git a/src/api/src/defines.rs b/src/api/src/defines.rs new file mode 100644 index 0000000..0b6f1ae --- /dev/null +++ b/src/api/src/defines.rs @@ -0,0 +1,35 @@ +// Copyright (c) Bao Project and Contributors. All rights reserved. +// João Peixoto +// +// SPDX-License-Identifier: Apache-2.0 + +//! Bao defines. + +#![allow(dead_code)] + +/// Bao I/O Write Operation +pub const BAO_IO_WRITE: u64 = 0x0; +/// Bao I/O Read Operation +pub const BAO_IO_READ: u64 = 0x1; +/// Bao I/O Ask Operation +pub const BAO_IO_ASK: u64 = 0x2; +/// Bao I/O Notify Operation +pub const BAO_IO_NOTIFY: u64 = 0x3; + +/// Bao Maximum Name Length +pub const BAO_NAME_LEN: usize = 16; + +/// Bao Maximum I/O Requests +pub const BAO_IO_REQUEST_MAX: usize = 16; + +/// Bao IOCTL Type +pub const BAO_IOCTL_TYPE: u32 = 0xA6; + +/// Bao I/O Event File Descriptor Data Match Flag +pub const BAO_IOEVENTFD_FLAG_DATAMATCH: u32 = 1 << 1; +/// Bao I/O Event File Descriptor Deassign Flag +pub const BAO_IOEVENTFD_FLAG_DEASSIGN: u32 = 1 << 2; +/// Bao IRQ File Descriptor Assign Flag +pub const BAO_IRQFD_FLAG_ASSIGN: u32 = 0x00; +/// Bao IRQ File Descriptor Deassign Flag +pub const BAO_IRQFD_FLAG_DEASSIGN: u32 = 0x01; diff --git a/src/api/src/device_model.rs b/src/api/src/device_model.rs new file mode 100644 index 0000000..a0f1ecf --- /dev/null +++ b/src/api/src/device_model.rs @@ -0,0 +1,300 @@ +// Copyright (c) Bao Project and Contributors. All rights reserved. +// João Peixoto +// +// SPDX-License-Identifier: Apache-2.0 + +//! Bao device model. + +use crate::defines::{BAO_IO_ASK, BAO_IRQFD_FLAG_ASSIGN}; +use crate::error::{Error, Result}; +use crate::ioctl::*; +use crate::types::{BaoIoEventFd, BaoIoRequest, BaoIrqFd}; +use libc::ioctl; +use std::os::fd::AsRawFd; +use vmm_sys_util::errno; +use vmm_sys_util::eventfd::EventFd; + +/// Bao Hypervisor Device Model. +/// +/// # Attributes +/// +/// * `fd` - The file descriptor for the VMM. +/// * `devmodel_fd` - The file descriptor for the device model. +/// * `id` - The ID of the device model. +#[derive(Clone)] +pub struct BaoDeviceModel { + pub fd: i32, + pub devmodel_fd: i32, + pub id: u16, +} + +impl BaoDeviceModel { + /// Create a new device model. + /// + /// # Arguments + /// + /// * `fd` - The file descriptor for the VMM. + /// * `id` - The ID of the device model. + /// + /// # Returns + /// + /// A `Result` containing the result of the operation. + pub fn new(fd: i32, id: u16) -> Result { + let devmodel_fd; + + unsafe { + devmodel_fd = ioctl(fd, BAO_IOCTL_VM_VIRTIO_BACKEND_CREATE(), &(id as i32)); + + if devmodel_fd < 0 { + return Err(Error::OpenFdFailed( + "devmodel_fd", + std::io::Error::last_os_error(), + )); + } + } + + // Create the device model object. + let device_model = BaoDeviceModel { + fd, + devmodel_fd, + id, + }; + + // Create the I/O client. + device_model.create_io_client().unwrap(); + + Ok(device_model) + } + + /// Destroy the device model. + /// + /// # Returns + /// + /// A `Result` containing the result of the operation. + pub fn destroy(&self) -> Result<()> { + unsafe { + let ret = ioctl( + self.devmodel_fd, + BAO_IOCTL_VM_VIRTIO_BACKEND_DESTROY(), + &(self.id as i32), + ); + + if ret < 0 { + return Err(Error::OpenFdFailed( + "guest_fd", + std::io::Error::last_os_error(), + )); + } + } + Ok(()) + } + + /// Create an I/O client. + /// + /// # Returns + /// + /// A `Result` containing the result of the operation. + fn create_io_client(&self) -> Result<()> { + unsafe { + let ret = ioctl(self.devmodel_fd, BAO_IOCTL_IO_CREATE_CLIENT()); + + if ret < 0 { + return Err(Error::BaoIoctlError( + std::io::Error::last_os_error(), + std::any::type_name::(), + )); + } + } + Ok(()) + } + + /// Destroy an I/O client. + /// + /// # Returns + /// + /// A `Result` containing the result of the operation. + pub fn destroy_io_client(&self) -> Result<()> { + unsafe { + let ret = ioctl(self.devmodel_fd, BAO_IOCTL_IO_DESTROY_CLIENT()); + + if ret < 0 { + return Err(Error::BaoIoctlError( + std::io::Error::last_os_error(), + std::any::type_name::(), + )); + } + } + Ok(()) + } + + /// Attach the I/O client to the VM. + /// + /// # Returns + /// + /// A `Result` containing the result of the operation. + pub fn attach_io_client(&self) -> Result<()> { + unsafe { + let ret = ioctl(self.devmodel_fd, BAO_IOCTL_IO_ATTACH_CLIENT()); + + if ret < 0 { + return Err(Error::BaoIoctlError( + std::io::Error::last_os_error(), + std::any::type_name::(), + )); + } + } + Ok(()) + } + + /// Requests an I/O request. + /// + /// # Return + /// + /// * `Result` - A Result containing the BaoIoRequest object on success. + pub fn request_io(&self) -> Result { + // Create a new I/O request + let mut request = BaoIoRequest { + virtio_id: 0, + reg_off: 0, + addr: 0, + op: BAO_IO_ASK, + value: 0, + access_width: 0, + cpu_id: 0, + vcpu_id: 0, + ret: 0, + }; + // Request the I/O request + unsafe { + let ret = ioctl(self.devmodel_fd, BAO_IOCTL_IO_REQUEST(), &mut request); + + if ret < 0 { + return Err(Error::BaoIoctlError( + std::io::Error::last_os_error(), + std::any::type_name::(), + )); + } + } + Ok(request) + } + + /// Notifies I/O request completion. + /// + /// # Arguments + /// + /// * `req` - The BaoIoRequest to be notified. + /// + /// # Return + /// + /// * `Result<()>` - A Result containing Ok(()) on success, or an Error on failure. + pub fn notify_io_completed(&self, req: BaoIoRequest) -> Result<()> { + // Notify I/O request completion + unsafe { + let ret = ioctl( + self.devmodel_fd, + BAO_IOCTL_IO_REQUEST_NOTIFY_COMPLETED(), + &req, + ); + + if ret < 0 { + return Err(Error::BaoIoctlError( + std::io::Error::last_os_error(), + std::any::type_name::(), + )); + } + } + + // Return Ok(()) on success + Ok(()) + } + + /// Notifies the guest about a Used Buffer Notification or + /// a Configuration Change Notification. + /// + /// # Return + /// + /// * `Result<()>` - A Result containing Ok(()) on success, or an Error on failure. + pub fn notify_guest(&self) -> Result<()> { + // Notify the guest + unsafe { + let ret = ioctl(self.devmodel_fd, BAO_IOCTL_IO_NOTIFY_GUEST()); + + if ret < 0 { + return Err(Error::BaoIoctlError( + std::io::Error::last_os_error(), + std::any::type_name::(), + )); + } + } + + // Return Ok(()) on success + Ok(()) + } + + /// Registers an ioeventfd within the VM (guest to host interrupt) + /// + /// # Arguments + /// + /// * `kick` - The EventFd to be registered. + /// * `flags` - The flags to be used. + /// * `addr` - The address to be registered. + /// * `datamatch` - The data to be matched (index of the Virtqueue). + /// + /// # Returns + /// + /// A `Result` containing the result of the operation. + pub fn register_ioeventfd( + &self, + kick: u32, + flags: u32, + addr: u64, + datamatch: u64, + ) -> Result<()> { + // Create a BaoIoEventFd struct. + let ioeventfd = BaoIoEventFd { + fd: kick, + flags: flags, + addr: addr, + len: 4, + reserved: 0, + data: datamatch, // Index of the Virtqueue to match with the 'value' field of the 'bao_io_request' struct + }; + + // Call the ioctl to register the ioeventfd. + unsafe { + let ret = ioctl(self.devmodel_fd, BAO_IOCTL_IOEVENTFD(), &ioeventfd); + + if ret < 0 { + return Err(Error::RegisterIoevent(errno::Error::last())); + } + } + Ok(()) + } + + /// Registers an irqfd within the VM (host to guest interrupt) + /// + /// # Arguments + /// + /// * `call` - The EventFd to be registered. + /// + /// # Returns + /// + /// A `Result` containing the result of the operation. + pub fn register_irqfd(&self, call: &EventFd) -> Result<()> { + // Create a BaoIrqFd struct. + let irqfd = BaoIrqFd { + fd: call.as_raw_fd() as i32, + flags: BAO_IRQFD_FLAG_ASSIGN, // Assign the Irqfd + }; + + // Call the ioctl to register the irqfd. + unsafe { + let ret = ioctl(self.devmodel_fd, BAO_IOCTL_IRQFD(), &irqfd); + + if ret < 0 { + return Err(Error::RegisterIrqfd(errno::Error::last())); + } + } + Ok(()) + } +} diff --git a/src/api/src/error.rs b/src/api/src/error.rs new file mode 100644 index 0000000..44c340c --- /dev/null +++ b/src/api/src/error.rs @@ -0,0 +1,92 @@ +// Copyright (c) Bao Project and Contributors. All rights reserved. +// João Peixoto +// +// SPDX-License-Identifier: Apache-2.0 + +//! Bao error cases. + +#![allow(dead_code)] + +use std::io::Error as IoError; +use std::{io, num::ParseIntError, str}; +use vmm_sys_util::errno; + +/// Result code. +pub type Result = std::result::Result; + +/// Error codes. +#[derive(Debug, thiserror::Error)] +pub enum Error { + #[error("Invalid Frontend ID {0:?}")] + InvalidFrontendId(u16), + #[error("Invalid MMIO {0:} Address {1:?}")] + InvalidMmioAddr(&'static str, u64), + #[error("MMIO Legacy not supported by Guest")] + MmioLegacyNotSupported, + #[error("IOMMU not supported by Guest")] + IommuPlatformNotSupported, + #[error("Invalid feature select {0:}")] + InvalidFeatureSel(u32), + #[error("Invalid MMIO direction {0:}")] + InvalidMmioDir(u8), + #[error("Device not supported: {0:}")] + BaoDevNotSupported(String), + #[error("Bao IOCTL error: {0:?} - {1:?}")] + BaoIoctlError(io::Error, &'static str), + #[error("Vhost user frontend error")] + VhostFrontendError(vhost_user_frontend::Error), + #[error("Vhost user frontend activate error")] + VhostFrontendActivateError(vhost_user_frontend::ActivateError), + #[error("Invalid String: {0:?}")] + InvalidString(str::Utf8Error), + #[error("Failed while parsing to integer: {0:?}")] + ParseFailure(ParseIntError), + #[error("Failed to create epoll context: {0:?}")] + EpollCreateFd(io::Error), + #[error("Failed to add event to epoll: {0:?}")] + RegisterExitEvent(io::Error), + #[error("Failed while waiting on epoll: {0:?}")] + EpollWait(io::Error), + #[error("Bao Bus Invalid State")] + BaoBusInvalidState, + #[error("Failed to kick backend: {0:?}")] + EventFdWriteFailed(io::Error), + #[error("Failed to open the file descriptor {0:?}: {1:?}")] + OpenFdFailed(&'static str, io::Error), + #[error("Invalid IO Request Direction: {0:?}")] + InvalidIoReqDirection(u64), + #[error("HandleIoEventFailed")] + HandleIoEventFailed, + #[error("Device not found")] + DeviceNotFound, + #[error("Mmap guest memory failed")] + MmapGuestMemoryFailed, + #[error("Failed to create event manager: {0:?}")] + EventManager(event_manager::Error), + #[error("Failed to register the Ioeventfd: {0:?}")] + RegisterIoevent(errno::Error), + #[error("Failed to register the Irqfd: {0:?}")] + RegisterIrqfd(errno::Error), + #[error("Failed to register the Mmio")] + MmioConfig, + #[error("Invalid MMIO {0:?} Operation")] + InvalidMmioOperation(&'static str), + #[error("Device not found: {0:?} - {1:?}")] + WrongDeviceConfiguration(String, String), + #[error("Device bad feaures: {0:?}")] + DeviceBadFeatures(u64), + #[error("Device already activated")] + DeviceAlreadyActivated, + #[error("Failed to create the VhostUserMemoryRegion")] + VhostUserMemoryRegion, + #[error("Failed to create the net tap device: {0:?}")] + NetTapCreateFailed(IoError), + #[error("Failed to open the net tap device")] + NetTapOpenFailed, + #[error("Failed to set the net interface name: {0:?}")] + NetInvalidIfname(String), + #[error("Failed to open /dev/net/tun: {0:?}")] + NetOpenTun(IoError), + #[error("Ioctl error: {0:?}")] + IoctlError(IoError), +} diff --git a/src/api/src/ioctl.rs b/src/api/src/ioctl.rs new file mode 100644 index 0000000..5d4a532 --- /dev/null +++ b/src/api/src/ioctl.rs @@ -0,0 +1,104 @@ +// Copyright (c) Bao Project and Contributors. All rights reserved. +// João Peixoto +// +// SPDX-License-Identifier: Apache-2.0 + +//! Bao IOCTLs. + +#![allow(dead_code)] + +use super::defines::BAO_IOCTL_TYPE; +use super::types::{BaoIoEventFd, BaoIoRequest, BaoIrqFd}; +use vmm_sys_util::ioctl::{_IOC_NONE, _IOC_READ, _IOC_WRITE}; +use vmm_sys_util::ioctl_ioc_nr; + +ioctl_ioc_nr!( + BAO_IOCTL_VM_VIRTIO_BACKEND_CREATE, + _IOC_WRITE, + BAO_IOCTL_TYPE, + 1 as u32, + std::mem::size_of::() as u32 +); +ioctl_ioc_nr!( + BAO_IOCTL_VM_VIRTIO_BACKEND_DESTROY, + _IOC_WRITE, + BAO_IOCTL_TYPE, + 2 as u32, + std::mem::size_of::() as u32 +); +ioctl_ioc_nr!( + BAO_IOCTL_IO_CREATE_CLIENT, + _IOC_NONE, + BAO_IOCTL_TYPE, + 3 as u32, + 0 +); +ioctl_ioc_nr!( + BAO_IOCTL_IO_DESTROY_CLIENT, + _IOC_NONE, + BAO_IOCTL_TYPE, + 4 as u32, + 0 +); +ioctl_ioc_nr!( + BAO_IOCTL_IO_ATTACH_CLIENT, + _IOC_NONE, + BAO_IOCTL_TYPE, + 5 as u32, + 0 +); +ioctl_ioc_nr!( + BAO_IOCTL_IO_REQUEST, + _IOC_WRITE | _IOC_READ, + BAO_IOCTL_TYPE, + 6 as u32, + std::mem::size_of::() as u32 +); +ioctl_ioc_nr!( + BAO_IOCTL_IO_REQUEST_NOTIFY_COMPLETED, + _IOC_WRITE, + BAO_IOCTL_TYPE, + 7 as u32, + std::mem::size_of::() as u32 +); +ioctl_ioc_nr!( + BAO_IOCTL_IO_NOTIFY_GUEST, + _IOC_NONE, + BAO_IOCTL_TYPE, + 8 as u32, + 0 +); +ioctl_ioc_nr!( + BAO_IOCTL_IOEVENTFD, + _IOC_WRITE, + BAO_IOCTL_TYPE, + 9 as u32, + std::mem::size_of::() as u32 +); +ioctl_ioc_nr!( + BAO_IOCTL_IRQFD, + _IOC_WRITE, + BAO_IOCTL_TYPE, + 10 as u32, + std::mem::size_of::() as u32 +); + +#[cfg(test)] +mod tests { + use super::*; + + /// Tests the BAO IOCTLs constants. + #[test] + fn test_ioctls() { + assert_eq!(0x4004_A601, BAO_IOCTL_VM_VIRTIO_BACKEND_CREATE()); + assert_eq!(0x4004_A602, BAO_IOCTL_VM_VIRTIO_BACKEND_DESTROY()); + assert_eq!(0x0000_A603, BAO_IOCTL_IO_CREATE_CLIENT()); + assert_eq!(0x0000_A604, BAO_IOCTL_IO_DESTROY_CLIENT()); + assert_eq!(0x0000_A605, BAO_IOCTL_IO_ATTACH_CLIENT()); + assert_eq!(0xC048_A606, BAO_IOCTL_IO_REQUEST()); + assert_eq!(0x4048_A607, BAO_IOCTL_IO_REQUEST_NOTIFY_COMPLETED()); + assert_eq!(0x0000_A608, BAO_IOCTL_IO_NOTIFY_GUEST()); + assert_eq!(0x4020_A609, BAO_IOCTL_IOEVENTFD()); + assert_eq!(0x4008_A60A, BAO_IOCTL_IRQFD()); + } +} diff --git a/src/api/src/lib.rs b/src/api/src/lib.rs new file mode 100644 index 0000000..45693b6 --- /dev/null +++ b/src/api/src/lib.rs @@ -0,0 +1,6 @@ +pub mod cli; +pub mod defines; +pub mod device_model; +pub mod error; +pub mod ioctl; +pub mod types; diff --git a/src/api/src/types.rs b/src/api/src/types.rs new file mode 100644 index 0000000..7bfdd69 --- /dev/null +++ b/src/api/src/types.rs @@ -0,0 +1,135 @@ +// Copyright (c) Bao Project and Contributors. All rights reserved. +// João Peixoto +// +// SPDX-License-Identifier: Apache-2.0 + +//! Bao custom types. + +#![allow(dead_code)] + +use serde::{Deserialize, Serialize}; + +/// Struct representing a Bao I/O request. +/// +/// # Attributes +/// +/// * `virtio_id` - Virtio instance ID. +/// * `reg_off` - Register offset. +/// * `addr` - Address. +/// * `op` - Operation. +/// * `value` - Value. +/// * `access_width` - Access width. +/// * `cpu_id` - Frontend CPU ID of the I/O request. +/// * `vcpu_id` - Frontend vCPU ID of the I/O request. +/// * `ret` - Return value. +#[repr(C)] +#[derive(Debug)] +pub struct BaoIoRequest { + pub virtio_id: u64, + pub reg_off: u64, + pub addr: u64, + pub op: u64, + pub value: u64, + pub access_width: u64, + pub cpu_id: u64, + pub vcpu_id: u64, + pub ret: u64, +} + +/// Struct representing a Bao I/O event file descriptor. +/// +/// # Attributes +/// +/// * `fd` - File descriptor. +/// * `flags` - Flags. +/// * `addr` - Address. +/// * `len` - Length. +/// * `reserved` - Reserved. +/// * `data` - Datamatch. +#[repr(C)] +pub struct BaoIoEventFd { + pub fd: u32, + pub flags: u32, + pub addr: u64, + pub len: u32, + pub reserved: u32, + pub data: u64, +} + +/// Struct representing a Bao IRQ file descriptor. +/// +/// # Attributes +/// +/// * `fd` - File descriptor. +/// * `flags` - Flags. +#[repr(C)] +pub struct BaoIrqFd { + pub fd: i32, + pub flags: u32, +} + +#[derive(Debug, Deserialize, Serialize, PartialEq)] +/// Struct representing a Device configuration. +/// +/// # Attributes +/// +/// * `id` - Device ID. +/// * `type` - Device type. +/// * `shmem_addr` - Shared memory address. +/// * `shmem_size` - Shared memory size. +/// * `shmem_path` - Shared memory path. +/// * `mmio_addr` - MMIO address. +/// * `irq` - Device interrupt. +/// * `data_plane` - Data plane type. +/// * `file_path` - File path (Block device specific option). +/// * `read_only` - Read only (Block device specific option). +/// * `root_device` - Root device (Block device specific option). +/// * `advertise_flush` - Advertise flush (Block device specific option). +/// * `tap_name` - TAP name (Network device specific option). +/// * `mac_addr` - MAC address (Network device specific option). +/// * `guest_cid` - Guest context ID (Vsock device specific option). +/// * `socket_path` - Socket path (Vhost-user device specific option). +pub struct DeviceConfig { + pub id: u32, + #[serde(rename = "type")] + pub device_type: String, + pub shmem_addr: u64, + pub shmem_size: u64, + pub shmem_path: String, + pub mmio_addr: u64, + pub irq: u32, + pub data_plane: String, + // Block device specific fields + pub file_path: Option, + pub read_only: Option, + pub root_device: Option, + pub advertise_flush: Option, + // Network device specific fields + pub tap_name: Option, + pub mac_addr: Option, + // Vsock device specific fields + pub guest_cid: Option, + // Vhost-user device specific fields + pub socket_path: Option, +} + +#[derive(Debug, Deserialize, Serialize, PartialEq)] +/// Struct representing the VMM configuration. +/// +/// # Attributes +/// +/// * `devices` - List of devices. +pub struct VMMConfig { + pub devices: Vec, +} + +/// An address either in programmable I/O space or in memory mapped I/O space. +/// +/// The `IoEventAddress` is used for specifying the type when registering an event +/// in [register_ioevent](struct.VmFd.html#method.register_ioevent). +pub enum IoEventAddress { + /// Representation of an programmable I/O address. + Pio(u64), + /// Representation of an memory mapped I/O address. + Mmio(u64), +} diff --git a/src/main.rs b/src/main.rs new file mode 100644 index 0000000..9e8748a --- /dev/null +++ b/src/main.rs @@ -0,0 +1,16 @@ +use api::cli::Cli; +use vmm::vmm::Vmm; + +fn main() { + // Create a new CLI object. + let cli = Cli::new(); + + // Launch the CI to parse the configuration file. + let vmm_config = cli.launch().unwrap(); + + // Create a new VMM. + let vmm = Vmm::try_from(vmm_config).unwrap(); + + // Run the VMM. + vmm.run().unwrap(); +} diff --git a/src/virtio/.gitignore b/src/virtio/.gitignore new file mode 100644 index 0000000..ea8c4bf --- /dev/null +++ b/src/virtio/.gitignore @@ -0,0 +1 @@ +/target diff --git a/src/virtio/Cargo.toml b/src/virtio/Cargo.toml new file mode 100644 index 0000000..fbe056d --- /dev/null +++ b/src/virtio/Cargo.toml @@ -0,0 +1,27 @@ +[package] +name = "virtio" +version = "0.1.0" +edition = "2021" +authors = ["João Peixoto "] +keywords = ["virtio", "vhost", "vhost-user"] +description = "VirtIO devices" +license = "Apache-2.0" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +event-manager = { version = "0.4.0", features = ["remote_endpoint"] } +vm-device = "0.1.0" +virtio-queue = "0.11.0" +virtio-bindings = "0.2.1" +vmm-sys-util = "0.12.1" +virtio-device = { git = "https://github.com/joaopeixoto13/vm-virtio", branch = "bao-hypervisor" } +virtio-blk = { git = "https://github.com/joaopeixoto13/vm-virtio", branch = "bao-hypervisor", features = ["backend-stdio"] } +virtio-vsock = { git = "https://github.com/joaopeixoto13/vm-virtio", branch = "bao-hypervisor" } +vhost = { version = "0.10.0", features = ["vhost-user-frontend", "vhost-kern", "vhost-user-backend", "vhost-vsock", "vhost-net"] } +vhost-user-frontend = { git = "https://github.com/joaopeixoto13/vhost", branch = "vhost-user-frontend" } +vm-memory = { version = "0.14.0", features = ["backend-mmap", "backend-atomic", "backend-bitmap"] } +api = { path = "../api" } +libc = ">=0.2.95" +seccompiler = "0.2.0" +log = "0.4.17" diff --git a/src/virtio/src/block/README.md b/src/virtio/src/block/README.md new file mode 100644 index 0000000..a3e4cb8 --- /dev/null +++ b/src/virtio/src/block/README.md @@ -0,0 +1,19 @@ +# VirtIO Block + +## Overview +The VirtIO Block device is a virtual block device specification designed for efficient and high-performance +block device access within virtualized environments. It enables virtual machines (VMs) to interact with +block storage devices, such as hard drives and SSDs, through a standardized interface, optimizing performance +and resource utilization. The frontend driver, in the frontend VM, places read, write, and other requests +onto the virtqueue, so that the backend driver, in the backend VM, can process them accordingly. +Communication between the frontend and backend is based on the virtio kick and notify mechanism. + +## Purpose + +The primary purpose of the VirtIO Block device within the context of Bao Hypervisor is to provide +VMs with access to virtual block storage devices, enhancing its storage virtualization capabilities, +and enabling VMs to efficiently interact with block storage devices for non-volatile +data storage and retrieval. + +## Requirements +- VirtIO Block support on the Frontend VM (e.g. `CONFIG_VIRTIO_BLK` on buildroot) \ No newline at end of file diff --git a/src/virtio/src/block/mod.rs b/src/virtio/src/block/mod.rs new file mode 100644 index 0000000..d43f304 --- /dev/null +++ b/src/virtio/src/block/mod.rs @@ -0,0 +1 @@ +pub mod virtio; diff --git a/src/virtio/src/block/virtio/README.md b/src/virtio/src/block/virtio/README.md new file mode 100644 index 0000000..133d01c --- /dev/null +++ b/src/virtio/src/block/virtio/README.md @@ -0,0 +1,33 @@ +# VirtIO Block + +## Quick start + +Follow these steps to quickly set up and run the VirtIO Block device with Bao Hypervisor: + +1. **Prepare Configuration File**: Create a configuration file (e.g. *config-virtio-block.yaml*) specifying +the settings for the virtio block device. One example of a configuration file could be: + +``` +devices: + # --- VirtIO Common --- + - id: 0 + type: "block" + shmem_addr: 0x50000000 + shmem_size: 0x01000000 + shmem_path: "/dev/baoipc0" + mmio_addr: 0xa003e00 + irq: 47 + data_plane: virtio + # --- Virtio Block Specific --- + file_path: "/etc/block.img" + read_only: false + root_device: true + advertise_flush: false + # ----------------------------- +``` + +2. Launch the **device model** with VirtIO Block device: + +``` +nohup bao-virtio --config /PATH/TO/YOUR/config-virtio-block.yaml > /etc/bao-virtio.log 2>&1 & +``` \ No newline at end of file diff --git a/src/virtio/src/block/virtio/device.rs b/src/virtio/src/block/virtio/device.rs new file mode 100644 index 0000000..1f57866 --- /dev/null +++ b/src/virtio/src/block/virtio/device.rs @@ -0,0 +1,252 @@ +use crate::device::{VirtioDevType, VirtioDeviceCommon}; +use std::fs::{File, OpenOptions}; +use std::io::{Seek, SeekFrom}; +use std::path::PathBuf; + +use super::inorder_handler::InOrderQueueHandler; +use super::queue_handler::QueueHandler; +use crate::device::{SingleFdSignalQueue, VirtioDeviceT}; +use api::device_model::BaoDeviceModel; +use api::error::{Error, Result}; +use api::types::DeviceConfig; +use event_manager::{EventManager, MutEventSubscriber}; +use log::error; +use std::borrow::{Borrow, BorrowMut}; +use std::cmp; +use std::sync::atomic::AtomicU8; +use std::sync::{Arc, Mutex}; +use virtio_bindings::virtio_blk::{VIRTIO_BLK_F_FLUSH, VIRTIO_BLK_F_RO}; +use virtio_blk::stdio_executor::StdIoBackend; +use virtio_device::{VirtioConfig, VirtioDeviceActions, VirtioDeviceType, VirtioMmioDevice}; +use virtio_queue::Queue; +use vm_device::bus::MmioAddress; +use vm_device::device_manager::{IoManager, MmioManager}; +use vm_device::MutDeviceMmio; + +// The sector size is 512 bytes (1 << 9). +const SECTOR_SHIFT: u8 = 9; + +/// Virtio block device. +/// +/// # Attributes +/// +/// * `common` - Virtio common device. +/// * `file_path` - Path to the block device file or disk partition. +/// * `read_only` - Whether the block device is read-only. +/// * `root_device` - Whether the block device is the root device. +/// * `advertise_flush` - Whether the block device advertises the flush feature. +pub struct VirtioBlock { + pub common: VirtioDeviceCommon, + pub file_path: PathBuf, + pub read_only: bool, + pub root_device: bool, + pub advertise_flush: bool, +} + +impl VirtioDeviceT for VirtioBlock { + fn new( + config: &DeviceConfig, + device_manager: Arc>, + event_manager: Arc>>>>, + device_model: Arc>, + ) -> Result>> { + // Extract the generic features and queues. + let (common_features, queues) = Self::initialize(&config).unwrap(); + + // Update the device features. + let device_features = common_features | Self::device_features(&config).unwrap(); + + // Update the configuration space. + let config_space = Self::config_space(&config).unwrap(); + + // Create a VirtioConfig object. + let virtio_cfg = VirtioConfig::new(device_features, queues, config_space); + + // Create the generic device. + let common_device = + VirtioDeviceCommon::new(config, event_manager, device_model, virtio_cfg).unwrap(); + + // Create the block device. + let block = Arc::new(Mutex::new(VirtioBlock { + common: common_device, + file_path: config.file_path.clone().unwrap().into(), + read_only: config.read_only.unwrap(), + root_device: config.root_device.unwrap(), + advertise_flush: config.advertise_flush.unwrap(), + })); + + // Register the MMIO device within the device manager with the specified range. + device_manager + .lock() + .unwrap() + .register_mmio( + block.clone().lock().unwrap().common.mmio.range, + block.clone(), + ) + .unwrap(); + + // Return the block device. + Ok(block) + } + + fn device_features(config: &DeviceConfig) -> Result { + let mut features = 0; + + // Set the read-only feature. + if config.read_only.unwrap() { + features |= 1 << VIRTIO_BLK_F_RO; + } + + // Set the flush feature. + if config.advertise_flush.unwrap() { + features |= 1 << VIRTIO_BLK_F_FLUSH; + } + + Ok(features) + } + + fn config_space(config: &DeviceConfig) -> Result> { + // TODO: right now, the file size is computed by the StdioBackend as well. Maybe we should + // create the backend as early as possible, and get the size information from there. + let file_size = File::open(config.file_path.clone().unwrap()) + .unwrap() + .seek(SeekFrom::End(0)) + .unwrap(); + + // If the file size is actually not a multiple of sector size, then data at the very end + // will be ignored. + let num_sectors = file_size >> SECTOR_SHIFT; + + // Update the configuration space. + // This must be little-endian according to the Virtio specification. + Ok(num_sectors.to_le_bytes().to_vec()) + } +} + +impl Borrow> for VirtioBlock { + fn borrow(&self) -> &VirtioConfig { + &self.common.config + } +} + +impl BorrowMut> for VirtioBlock { + fn borrow_mut(&mut self) -> &mut VirtioConfig { + &mut self.common.config + } +} + +impl VirtioDeviceType for VirtioBlock { + fn device_type(&self) -> u32 { + VirtioDevType::Block as u32 + } +} + +/// Implement the `VirtioDeviceActions` trait to add our custom device actions. +impl VirtioDeviceActions for VirtioBlock { + type E = Error; + + fn activate(&mut self) -> Result<()> { + // Open the block device file. + let file = OpenOptions::new() + .read(true) + .write(!self.read_only) + .open(&self.file_path) + .unwrap(); + + // Create the backend. + // TODO: Create the backend earlier (as part of `VirtioBlock::new`)? + let disk = StdIoBackend::new(file, self.common.config.driver_features).unwrap(); + + // Create the driver notify object. + let driver_notify = SingleFdSignalQueue { + irqfd: self.common.irqfd.try_clone().unwrap(), + interrupt_status: self.common.config.interrupt_status.clone(), + }; + + // Prepare the activation by calling the generic `prepare_activate` method. + let mut ioevents = self.common.prepare_activate().unwrap(); + + // Create the inner handler. + let inner = InOrderQueueHandler { + driver_notify, + mem: self.common.mem(), + queue: self.common.config.queues.remove(0), + disk, + }; + + // Create the queue handler. + let handler = Arc::new(Mutex::new(QueueHandler { + inner, + ioeventfd: ioevents.remove(0), + })); + + // Finalize the activation by calling the generic `finalize_activate` method. + let ret = self.common.finalize_activate(handler); + + Ok(ret.unwrap()) + } + + fn reset(&mut self) -> Result<()> { + // Not implemented for now. + Ok(()) + } + + fn read_config(&self, offset: usize, data: &mut [u8]) { + let config_space = &self.common.config.config_space; + let config_len = config_space.len(); + if offset >= config_len { + error!("Failed to read from config space"); + return; + } + + // TODO: Are partial reads ok? + let end = cmp::min(offset.saturating_add(data.len()), config_len); + let read_len = end - offset; + // Cannot fail because the lengths are identical and we do bounds checking beforehand. + data[..read_len].copy_from_slice(&config_space[offset..end]) + } + + fn write_config(&mut self, offset: usize, data: &[u8]) { + let config_space = &mut self.common.config.config_space; + let config_len = config_space.len(); + if offset >= config_len { + error!("Failed to write to config space"); + return; + } + + // TODO: Are partial writes ok? + let end = cmp::min(offset.saturating_add(data.len()), config_len); + let write_len = end - offset; + // Cannot fail because the lengths are identical and we do bounds checking beforehand. + config_space[offset..end].copy_from_slice(&data[..write_len]); + } + + fn negotiate_driver_features(&mut self) { + // Do nothing here since the features are already negotiated. + } + + fn interrupt_status(&self) -> &Arc { + // Simply return the interrupt status, since the backend (that is inside the VMM) will + // update it. + &self.common.config.interrupt_status + } +} + +/// Implement the `VirtioMmioDevice` trait to add VirtIO MMIO support to our device. +impl VirtioMmioDevice for VirtioBlock { + fn queue_notify(&mut self, _val: u32) { + // Do nothing for now. + } +} + +/// Implement the `DeviceMmio` mutable trait to add MMIO support to our device. +/// Otherwise we could not register the device within the device manager. +impl MutDeviceMmio for VirtioBlock { + fn mmio_read(&mut self, _base: MmioAddress, offset: u64, data: &mut [u8]) { + self.read(offset, data); + } + + fn mmio_write(&mut self, _base: MmioAddress, offset: u64, data: &[u8]) { + self.write(offset, data); + } +} diff --git a/src/virtio/src/block/virtio/inorder_handler.rs b/src/virtio/src/block/virtio/inorder_handler.rs new file mode 100644 index 0000000..997ca63 --- /dev/null +++ b/src/virtio/src/block/virtio/inorder_handler.rs @@ -0,0 +1,98 @@ +use crate::device::SignalUsedQueue; +use std::fs::File; +use std::result; +use virtio_blk::request::Request; +use virtio_blk::stdio_executor::{self, StdIoBackend}; +use virtio_queue::{DescriptorChain, Queue, QueueOwnedT, QueueT}; +use vm_memory::bitmap::AtomicBitmap; + +type GuestMemoryMmap = vm_memory::GuestMemoryMmap; + +pub struct InOrderQueueHandler { + pub driver_notify: S, + pub mem: GuestMemoryMmap, + pub queue: Queue, + pub disk: StdIoBackend, +} + +impl InOrderQueueHandler +where + S: SignalUsedQueue, +{ + /// Process a chain. + fn process_chain( + &mut self, + mut chain: DescriptorChain<&GuestMemoryMmap>, + ) -> result::Result<(), Error> { + let used_len = match Request::parse(&mut chain) { + // Process the backend request. + Ok(request) => self.disk.process_request(chain.memory(), &request)?, + Err(e) => { + println!("block request parse error: {:?}", e); + 0 + } + }; + + // Add the used descriptor to the queue. + self.queue + .add_used(chain.memory(), chain.head_index(), used_len)?; + + // Signal the driver, if needed. + if self.queue.needs_notification(chain.memory())? { + self.driver_notify.signal_used_queue(0); + } + + Ok(()) + } + + /// Process the queue. + /// + /// # Returns + /// + /// * `()` - Ok if the queue was processed successfully. + pub fn process_queue(&mut self) -> result::Result<(), Error> { + // To see why this is done in a loop, please look at the `Queue::enable_notification` + // comments in `virtio_queue`. + loop { + // Disable the notifications. + self.queue.disable_notification(&self.mem)?; + + // Process the queue. + while let Some(chain) = self.queue.iter(&self.mem.clone())?.next() { + self.process_chain(chain)?; + } + + // Enable the notifications. + if !self.queue.enable_notification(&self.mem)? { + break; + } + } + + Ok(()) + } +} + +#[derive(Debug)] +pub enum Error { + GuestMemory(vm_memory::GuestMemoryError), + Queue(virtio_queue::Error), + ProcessRequest(stdio_executor::ProcessReqError), +} + +impl From for Error { + fn from(e: vm_memory::GuestMemoryError) -> Self { + Error::GuestMemory(e) + } +} + +impl From for Error { + fn from(e: virtio_queue::Error) -> Self { + Error::Queue(e) + } +} + +impl From for Error { + fn from(e: stdio_executor::ProcessReqError) -> Self { + Error::ProcessRequest(e) + } +} diff --git a/src/virtio/src/block/virtio/mod.rs b/src/virtio/src/block/virtio/mod.rs new file mode 100644 index 0000000..7623ee5 --- /dev/null +++ b/src/virtio/src/block/virtio/mod.rs @@ -0,0 +1,3 @@ +pub mod device; +pub mod inorder_handler; +pub mod queue_handler; diff --git a/src/virtio/src/block/virtio/queue_handler.rs b/src/virtio/src/block/virtio/queue_handler.rs new file mode 100644 index 0000000..b375827 --- /dev/null +++ b/src/virtio/src/block/virtio/queue_handler.rs @@ -0,0 +1,53 @@ +use event_manager::{EventOps, Events, MutEventSubscriber}; +use vmm_sys_util::epoll::EventSet; +use vmm_sys_util::eventfd::EventFd; + +use crate::block::virtio::inorder_handler::InOrderQueueHandler; +use crate::device::SingleFdSignalQueue; + +const IOEVENT_DATA: u32 = 0; + +// This object simply combines the more generic `InOrderQueueHandler` with a concrete queue +// signalling implementation based on `EventFd`s, and then also implements `MutEventSubscriber` +// to interact with the event manager. `ioeventfd` is the `EventFd` connected to queue +// notifications coming from the driver. +pub(crate) struct QueueHandler { + pub inner: InOrderQueueHandler, + pub ioeventfd: EventFd, +} + +/// Implement the `MutEventSubscriber` trait for `QueueHandler` to handle the dispatched +/// events (Ioeventfds) from the event manager. +impl MutEventSubscriber for QueueHandler { + fn process(&mut self, events: Events, ops: &mut EventOps) { + let mut error = true; + + // TODO: Have a look at any potential performance impact caused by these conditionals + // just to be sure. + if events.event_set() != EventSet::IN { + println!("unexpected event_set"); + } else if events.data() != IOEVENT_DATA { + println!("unexpected events data {}", events.data()); + } else if self.ioeventfd.read().is_err() { + println!("ioeventfd read error") + } else if let Err(e) = self.inner.process_queue() { + println!("error processing block queue {:?}", e); + } else { + error = false; + } + + if error { + ops.remove(events) + .expect("Failed to remove fd from event handling loop"); + } + } + + fn init(&mut self, ops: &mut EventOps) { + ops.add(Events::with_data( + &self.ioeventfd, + IOEVENT_DATA, + EventSet::IN, + )) + .expect("Failed to init block queue handler"); + } +} diff --git a/src/virtio/src/device.rs b/src/virtio/src/device.rs new file mode 100644 index 0000000..597dfb9 --- /dev/null +++ b/src/virtio/src/device.rs @@ -0,0 +1,604 @@ +use super::block::virtio::device::VirtioBlock; +use super::fs::vhost_user::device::VhostUserFs; +use super::mmio::MmioConfig; +use super::mmio::VIRTIO_MMIO_INT_VRING; +use super::mmio::VIRTIO_MMIO_QUEUE_NOTIFY_OFFSET; +use super::net::vhost::device::VhostNet; +use super::net::virtio::device::VirtioNet; +use super::vsock::vhost::device::VhostVsockDevice; +use super::vsock::vhost_user::device::VhostUserVsock; +use api::defines::BAO_IOEVENTFD_FLAG_DATAMATCH; +use api::device_model::BaoDeviceModel; +use api::error::{Error, Result}; +use api::types::DeviceConfig; +use event_manager::{ + EventManager, MutEventSubscriber, RemoteEndpoint, Result as EvmgrResult, SubscriberId, +}; +use libc::{MAP_SHARED, PROT_READ, PROT_WRITE}; +use std::fmt::{self, Debug}; +use std::fs::OpenOptions; +use std::os::fd::AsRawFd; +use std::sync::atomic::{AtomicU8, Ordering}; +use std::sync::{Arc, Mutex}; +use vhost_user_frontend::{GuestMemoryMmap, GuestRegionMmap}; +use virtio_device::VirtioConfig; +use virtio_queue::{Queue, QueueT}; +use vm_device::device_manager::IoManager; +use vm_memory::{guest_memory::FileOffset, GuestAddress, MmapRegion}; +use vmm_sys_util::eventfd::{EventFd, EFD_NONBLOCK}; + +use virtio_bindings::virtio_config::{ + VIRTIO_F_IN_ORDER, VIRTIO_F_IOMMU_PLATFORM, VIRTIO_F_VERSION_1, +}; + +/// This feature enables the used_event and the avail_event (Notification Suppression). +pub const VIRTIO_F_RING_EVENT_IDX: u32 = 29; + +/// Type alias for the subscriber. +pub type Subscriber = Arc>; + +// Clippy thinks that values of the enum are too different in size. +#[allow(clippy::large_enum_variant)] +/// Virtio device type abstraction to pack all possible devices into one enum. +pub enum VirtioDeviceType { + VirtioBlock(Arc>), + VhostUserFs(Arc>), + VhostVsock(Arc>), + VhostNet(Arc>), + VhostUserVsock(Arc>), + VirtioNet(Arc>), + Unknown, +} + +/// VirtioDeviceCommon struct. +/// +/// # Attributes +/// +/// * `config` - The common virtio configuration. +/// * `mmio` - The MMIO configuration. +/// * `endpoint` - The remote subscriber endpoint. +/// * `irqfd` - The interrupt file descriptor. +/// * `device_model` - The device model. +/// * `regions` - The memory regions of the device. +pub struct VirtioDeviceCommon { + pub config: VirtioConfig, + pub mmio: MmioConfig, + pub endpoint: RemoteEndpoint, + pub irqfd: EventFd, + pub device_model: Arc>, + pub regions: Vec, +} + +impl VirtioDeviceCommon { + /// Create a new device. + /// + /// # Arguments + /// + /// * `config` - The device configuration. + /// * `device_manager` - The device manager. + /// * `event_manager` - The event manager. + /// * `device_model` - The device model. + /// + /// # Returns + /// + /// A `Result` containing the new device. + pub fn new( + config: &DeviceConfig, + event_manager: Arc>>>>, + device_model: Arc>, + virtio: VirtioConfig, + ) -> Result { + // Create the MMIO configuration. + let mmio = MmioConfig::new(config.mmio_addr, 0x200, config.irq).unwrap(); + + // Create a remote endpoint object, that allows interacting with the VM EventManager from a different thread. + // This is only needed for the Virtio data plane, since the Vhost and VhostUser data planes do not need to interact with the EventManager + // (the backend handler is outside of the VMM). + let remote_endpoint = event_manager.lock().unwrap().remote_endpoint(); + + // Create a new EventFd for the interrupt (irqfd). + let irqfd = EventFd::new(0).unwrap(); + + // Create the device object. + let mut device = VirtioDeviceCommon { + config: virtio, + mmio, + endpoint: remote_endpoint, + irqfd: irqfd, + device_model, + regions: Vec::new(), + }; + + // Map the region. + // The mmap_offset is set to 0 because the base address of Bao's shared memory driver is + // already defined statically in the backend device tree. + device + .map_region( + 0, + &config.shmem_path, + config.shmem_addr, + config.shmem_size as usize, + ) + .unwrap(); + + // Register the Irqfd (Host to Guest notification). + device + .device_model + .lock() + .unwrap() + .register_irqfd(&device.irqfd) + .unwrap(); + + // Return the device object. + Ok(device) + } + + /// Perform common initial steps for device activation based on the configuration + /// like setting up the event file descriptors for guest to host notifications. + /// + /// # Returns + /// + /// A `Result` containing the event file descriptors. + pub fn prepare_activate(&self) -> Result> { + // Check if the device has already been activated. + if self.config.device_activated { + return Err(Error::DeviceAlreadyActivated); + } + + // We do not support legacy drivers. + if self.config.driver_features & (1 << VIRTIO_F_VERSION_1) == 0 { + return Err(Error::DeviceBadFeatures(self.config.driver_features)); + } + + // Create an empty vector to store all event file descriptors. + let mut ioevents = Vec::new(); + + // Right now, we operate under the assumption all queues are marked ready by the device + // (which is true until we start supporting devices that can optionally make use of + // additional queues on top of the defaults). + for (i, _queue) in self.config.queues.iter().enumerate() { + // Create a new EventFd for the queue (Ioeventfd -> Guest to Host notification). + let fd = EventFd::new(EFD_NONBLOCK).unwrap(); + + // Register the queue event fd. + self.device_model + .lock() + .unwrap() + .register_ioeventfd( + fd.as_raw_fd() as u32, + BAO_IOEVENTFD_FLAG_DATAMATCH, + self.mmio.range.base().0 + VIRTIO_MMIO_QUEUE_NOTIFY_OFFSET, + // The maximum number of queues should fit within an `u16` according to the + // standard, so the conversion below is always expected to succeed. + i as u64, + ) + .unwrap(); + + ioevents.push(fd); + } + + Ok(ioevents) + } + + /// Perform the final steps of device activation based on the inner configuration and the + /// provided subscriber that's going to handle the device queues. + /// + /// Note: This method is unnecessary for the Vhost and VhostUser data planes since the + /// backend handler is outside of the VMM. + /// + /// # Arguments + /// + /// * `handler` - The subscriber that's going to handle the device queues. + /// + /// # Returns + /// + /// A `Result` containing operation result. + pub fn finalize_activate(&mut self, handler: Subscriber) -> Result<()> { + // Register the queue handler with the `EventManager`. We could record the `sub_id` + // (and/or keep a handler clone) for further interaction (i.e. to remove the subscriber at + // a later time, retrieve state, etc). + let _sub_id = self + .endpoint + .call_blocking(move |mgr| -> EvmgrResult { + Ok(mgr.add_subscriber(handler)) + }) + .unwrap(); + + // Set the device as activated. + self.config.device_activated = true; + + Ok(()) + } + + /// Method to map a region. + /// + /// # Arguments + /// + /// * `mmap_offset` - Offset of the mmap region. + /// * `path` - Path to the file. + /// * `base_addr` - Base address of the region. + /// * `size` - Size of the region. + /// + /// # Returns + /// + /// * `Result<()>` - A Result containing Ok(()) on success, or an Error on failure. + fn map_region( + &mut self, + mmap_offset: u64, + path: &str, + base_addr: u64, + size: usize, + ) -> Result<()> { + // Open the file. + let file = OpenOptions::new() + .read(true) + .write(true) + .open(path) + .unwrap(); + + // Create a mmap region with proper permissions. + let mmap_region = match MmapRegion::build( + Some(FileOffset::new(file, mmap_offset)), + base_addr as usize + size as usize, + PROT_READ | PROT_WRITE, + MAP_SHARED, + ) { + Ok(mmap_region) => mmap_region, + Err(_) => { + return Err(Error::MmapGuestMemoryFailed); + } + }; + + // Create a guest region mmap. + let guest_region_mmap = match GuestRegionMmap::new(mmap_region, GuestAddress(base_addr)) { + Ok(guest_region_mmap) => guest_region_mmap, + Err(_) => { + return Err(Error::MmapGuestMemoryFailed); + } + }; + + // Push the region to the regions vector. + // For now, we only have one region since this function is called only once. + // However, in the future, we may have to support more than one region. + self.regions.push(guest_region_mmap); + + // Return the guest region mmap. + Ok(()) + } + + /// Method to get the memory of the device. + /// + /// # Returns + /// + /// * `GuestMemoryMmap` - Guest memory mmap. + pub fn mem(&mut self) -> GuestMemoryMmap { + // Create a new GuestMemoryMmap from the regions without removing them. + GuestMemoryMmap::from_regions(self.regions.drain(..).collect()).unwrap() + } +} + +/// Trait to model the common virtio device operations. +/// Each virtio device type should implement this trait. +pub trait VirtioDeviceT { + /// Initialize the generic device. + /// + /// # Arguments + /// + /// * `config` - The device configuration. + /// + /// # Returns + /// + /// A `Result` containing the generic device features and the queues. + fn initialize(config: &DeviceConfig) -> Result<(u64, Vec)> { + // Extract the device type. + let device_type = VirtioDevType::from(config.device_type.as_str()); + + // Extract the number of queues and queue size for the device type. + let (queue_num, queue_size) = device_type.queue_num_and_size(); + + // Create the queues. + let mut queues = Vec::with_capacity(queue_num); + for _ in 0..queue_num { + queues.push(Queue::new(queue_size as u16)); + } + + // Convert the vector of Result to a vector of Queue. + let queues_converted: Vec = queues + .into_iter() + .collect::, _>>() + .unwrap(); + + // Define the generic device features. + let device_features = 1 << VIRTIO_F_VERSION_1 | 1 << VIRTIO_F_IOMMU_PLATFORM | 1 << VIRTIO_F_IN_ORDER /*| 1 << VIRTIO_F_RING_EVENT_IDX*/; + + Ok((device_features, queues_converted)) + } + + /// Create a new device. + /// + /// # Arguments + /// + /// * `config` - The device configuration. + /// * `device_manager` - The device manager. + /// * `event_manager` - The event manager. + /// * `device_model` - The device model. + /// + /// # Returns + /// + /// A `Result` containing the new device. + fn new( + config: &DeviceConfig, + device_manager: Arc>, + event_manager: Arc>>>>, + device_model: Arc>, + ) -> Result>>; + + /// Returns the specific device features. + /// + /// # Arguments + /// + /// * `config` - The device configuration. + /// + /// # Returns + /// + /// A `Result` containing the specific device features. + fn device_features(config: &DeviceConfig) -> Result; + + /// Returns the specific device configuration space. + /// + /// # Arguments + /// + /// * `config` - The device configuration. + /// + /// # Returns + /// + /// A `Result` containing the specific device configuration space. + fn config_space(config: &DeviceConfig) -> Result>; +} + +/// Simple trait to model the operation of signalling the driver about used events +/// for the specified queue. +pub trait SignalUsedQueue { + /// Signals the driver about used events for the specified queue. + fn signal_used_queue(&self, index: u16); +} + +/// Uses a single irqfd as the basis of signalling any queue (useful for the MMIO transport, +/// where a single interrupt is shared for everything). +/// +/// # Attributes +/// +/// * `irqfd` - The EventFd to be used for signalling. +/// * `interrupt_status` - The interrupt status to be used for signalling. +pub struct SingleFdSignalQueue { + pub irqfd: EventFd, + pub interrupt_status: Arc, +} + +impl SignalUsedQueue for SingleFdSignalQueue { + /// Signals the driver about used events for the specified queue. + fn signal_used_queue(&self, _index: u16) { + // Set the interrupt status. + self.interrupt_status + .fetch_or(VIRTIO_MMIO_INT_VRING, Ordering::SeqCst); + + // Write to the eventfd to signal the queue. + self.irqfd + .write(1) + .expect("Failed write to eventfd when signalling queue"); + } +} + +/// Virtio types taken from linux/virtio_ids.h +#[derive(Copy, Clone, Debug)] +#[allow(dead_code)] +#[allow(non_camel_case_types)] +#[repr(C)] +pub enum VirtioDevType { + Net = 1, + Block = 2, + Console = 3, + Rng = 4, + Balloon = 5, + Fs9P = 9, + Gpu = 16, + Input = 18, + Vsock = 19, + Iommu = 23, + Mem = 24, + Fs = 26, + Pmem = 27, + I2c = 34, + Watchdog = 35, // Temporary until official number allocated + Gpio = 41, + Unknown = 0xFF, +} + +impl From for VirtioDevType { + fn from(t: u32) -> Self { + match t { + 1 => VirtioDevType::Net, + 2 => VirtioDevType::Block, + 3 => VirtioDevType::Console, + 4 => VirtioDevType::Rng, + 5 => VirtioDevType::Balloon, + 9 => VirtioDevType::Fs9P, + 16 => VirtioDevType::Gpu, + 18 => VirtioDevType::Input, + 19 => VirtioDevType::Vsock, + 23 => VirtioDevType::Iommu, + 24 => VirtioDevType::Mem, + 26 => VirtioDevType::Fs, + 27 => VirtioDevType::Pmem, + 34 => VirtioDevType::I2c, + 35 => VirtioDevType::Watchdog, + 41 => VirtioDevType::Gpio, + _ => VirtioDevType::Unknown, + } + } +} + +impl From<&str> for VirtioDevType { + fn from(t: &str) -> Self { + match t { + "net" => VirtioDevType::Net, + "block" => VirtioDevType::Block, + "console" => VirtioDevType::Console, + "rng" => VirtioDevType::Rng, + "balloon" => VirtioDevType::Balloon, + "fs9p" => VirtioDevType::Fs9P, + "gpu" => VirtioDevType::Gpu, + "input" => VirtioDevType::Input, + "vsock" => VirtioDevType::Vsock, + "iommu" => VirtioDevType::Iommu, + "mem" => VirtioDevType::Mem, + "fs" => VirtioDevType::Fs, + "pmem" => VirtioDevType::Pmem, + "i2c" => VirtioDevType::I2c, + "watchdog" => VirtioDevType::Watchdog, + "gpio" => VirtioDevType::Gpio, + _ => VirtioDevType::Unknown, + } + } +} + +impl From for String { + fn from(t: VirtioDevType) -> String { + match t { + VirtioDevType::Net => "net", + VirtioDevType::Block => "block", + VirtioDevType::Console => "console", + VirtioDevType::Rng => "rng", + VirtioDevType::Balloon => "balloon", + VirtioDevType::Gpu => "gpu", + VirtioDevType::Fs9P => "9p", + VirtioDevType::Input => "input", + VirtioDevType::Vsock => "vsock", + VirtioDevType::Iommu => "iommu", + VirtioDevType::Mem => "mem", + VirtioDevType::Fs => "fs", + VirtioDevType::Pmem => "pmem", + VirtioDevType::I2c => "i2c", + VirtioDevType::Watchdog => "watchdog", + VirtioDevType::Gpio => "gpio", + VirtioDevType::Unknown => "UNKNOWN", + } + .to_string() + } +} + +// In order to use the `{}` marker, the trait `fmt::Display` must be implemented +// manually for the type VirtioDevType. +impl fmt::Display for VirtioDevType { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + write!( + f, + "{}", + match *self { + VirtioDevType::Net => String::from("Net"), + VirtioDevType::Block => String::from("Block"), + VirtioDevType::Console => String::from("Console"), + VirtioDevType::Rng => String::from("Rng"), + VirtioDevType::Balloon => String::from("Balloon"), + VirtioDevType::Gpu => String::from("Gpu"), + VirtioDevType::Fs9P => String::from("Fs9P"), + VirtioDevType::Input => String::from("Input"), + VirtioDevType::Vsock => String::from("Vsock"), + VirtioDevType::Iommu => String::from("Iommu"), + VirtioDevType::Mem => String::from("Mem"), + VirtioDevType::Fs => String::from("Fs"), + VirtioDevType::Pmem => String::from("Pmem"), + VirtioDevType::I2c => String::from("I2c"), + VirtioDevType::Watchdog => String::from("Watchdog"), + VirtioDevType::Gpio => String::from("Gpio"), + VirtioDevType::Unknown => String::from("Unknown"), + } + ) + } +} + +impl VirtioDevType { + /// Returns the number of queues and the queue size for the device type. + pub fn queue_num_and_size(&self) -> (usize, usize) { + match *self { + VirtioDevType::Net => (2, 1024), + VirtioDevType::Block => (1, 256), + VirtioDevType::Console => (0, 0), + VirtioDevType::Rng => (1, 1024), + VirtioDevType::Balloon => (0, 0), + VirtioDevType::Gpu => (0, 0), + VirtioDevType::Fs9P => (0, 0), + VirtioDevType::Input => (0, 0), + VirtioDevType::Vsock => (3, 1024), // Virtio spec says 3 queues, but vhost/vhost-user only manage 2 (RX/TX) + VirtioDevType::Mem => (0, 0), + VirtioDevType::Fs => (2, 1024), + VirtioDevType::Pmem => (0, 0), + VirtioDevType::I2c => (1, 1024), + VirtioDevType::Watchdog => (0, 0), + VirtioDevType::Gpio => (2, 256), + _ => (0, 0), + } + } +} + +/// Virtio data plane types. +#[derive(Copy, Clone, Debug)] +#[allow(dead_code)] +#[allow(non_camel_case_types)] +pub enum VirtioDataPlane { + Virtio = 0, + Vhost = 1, + VhostUser = 2, + Unknown = 0xFF, +} + +impl From for VirtioDataPlane { + fn from(t: u32) -> Self { + match t { + 0 => VirtioDataPlane::Virtio, + 1 => VirtioDataPlane::Vhost, + 2 => VirtioDataPlane::VhostUser, + _ => VirtioDataPlane::Unknown, + } + } +} + +impl From<&str> for VirtioDataPlane { + fn from(t: &str) -> Self { + match t { + "virtio" => VirtioDataPlane::Virtio, + "vhost" => VirtioDataPlane::Vhost, + "vhost_user" => VirtioDataPlane::VhostUser, + _ => VirtioDataPlane::Unknown, + } + } +} + +impl From for String { + fn from(t: VirtioDataPlane) -> String { + match t { + VirtioDataPlane::Virtio => "virtio", + VirtioDataPlane::Vhost => "vhost", + VirtioDataPlane::VhostUser => "vhost_user", + VirtioDataPlane::Unknown => "UNKNOWN", + } + .to_string() + } +} + +// In order to use the `{}` marker, the trait `fmt::Display` must be implemented +// manually for the type VirtioDataPlane. +impl fmt::Display for VirtioDataPlane { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + write!( + f, + "{}", + match *self { + VirtioDataPlane::Virtio => String::from("Virtio"), + VirtioDataPlane::Vhost => String::from("Vhost"), + VirtioDataPlane::VhostUser => String::from("VhostUser"), + VirtioDataPlane::Unknown => String::from("Unknown"), + } + ) + } +} diff --git a/src/virtio/src/fs/README.md b/src/virtio/src/fs/README.md new file mode 100644 index 0000000..0ca7f4f --- /dev/null +++ b/src/virtio/src/fs/README.md @@ -0,0 +1,21 @@ +# Virtual File System + +## Overview +The Virtual File System device is a virtualized file system designed to provide efficient and secure +access to volatile file storages within virtualized environments. It allows virtual machines (VMs) to +interact with host file systems and shared folders using a standardized interface or acting as a gateway +to a remote file system, facilitating seamless file I/O operations between VMs and the host system. +The device interface is based on the Linux Filesystem in Userspace (FUSE) protocol. The device acts as +the FUSE file system daemon and the driver acts as the FUSE client mounting the file system. The virtio +file system device provides the mechanism for transporting FUSE requests, much like */dev/fuse* in a +traditional FUSE application. + +## Purpose + +The primary purpose of the VirtIO File System device is to enable VMs to access file storage resources in +a virtualized environment with optimal performance and minimal overhead. By leveraging the VirtFS protocol, +VMs can mount host file systems or shared folders as virtual file systems, enabling them to read, write, +and manage files as if they were accessing local storage. + +## Requirements +- Virtual File System support on the Frontend VM (e.g. `CONFIG_VIRTIO_FS` on buildroot) \ No newline at end of file diff --git a/src/virtio/src/fs/mod.rs b/src/virtio/src/fs/mod.rs new file mode 100644 index 0000000..1a13971 --- /dev/null +++ b/src/virtio/src/fs/mod.rs @@ -0,0 +1 @@ +pub mod vhost_user; diff --git a/src/virtio/src/fs/vhost_user/README.md b/src/virtio/src/fs/vhost_user/README.md new file mode 100644 index 0000000..7cc7e23 --- /dev/null +++ b/src/virtio/src/fs/vhost_user/README.md @@ -0,0 +1,45 @@ +# Vhost-user virtual filesystem + +To provide Virtual Filesystem functionality, any vhost-user filesystem backend can be utilized. Nonetheless, for the purposes of demonstration, the [vhost-user virtio-fs device backend written in Rust](https://gitlab.com/virtio-fs/virtiofsd) was used. + + +## Quick start + +Follow these steps to quickly set up and run the Vhost-user virtual filesystem device with Bao Hypervisor: + +1. **Build the vhost-user virtual filesystem backend**: After adhering to all the instructions for constructing the virtiofs, it becomes necessary to append the following line to the end of `Cargo.toml` file and subsequently rebuild it: + +``` +[patch.crates-io] +vm-memory = { git = "https://github.com/joaopeixoto13/vm-memory", branch = "bao-hypervisor" } +``` + +2. **Prepare Configuration File**: Create a configuration file (e.g. *config-virtio-fs.yaml*) specifying +the settings for the vhost-user virtual filesystem device. One example of a configuration file could be: + +``` +devices: + # --- VirtIO Common --- + - id: 0 + type: "fs" + shmem_addr: 0x50000000 + shmem_size: 0x01000000 + shmem_path: "/dev/baoipc0" + mmio_addr: 0xa003e00 + irq: 47 + data_plane: vhost_user + # --- Vhost-user specific --- + socket_path: "/root/" + # ----------------------------- +``` + +3. Launch your **standalone vhost-user** virtual filesystem device: +``` +nohup virtiofsd --socket-path=/root/Fs.sock --shared-dir /mnt --tag=myfs --announce-submounts --sandbox chroot > /etc/vhost-device-virtiofsd.log 2>&1 & +``` + +4. Launch the **device model** with the vhost-user virtual filesystem frontend device: + +``` +nohup bao-virtio --config /PATH/TO/YOUR/config-virtio-fs.yaml > /etc/bao-virtio.log 2>&1 & +``` \ No newline at end of file diff --git a/src/virtio/src/fs/vhost_user/device.rs b/src/virtio/src/fs/vhost_user/device.rs new file mode 100644 index 0000000..a354863 --- /dev/null +++ b/src/virtio/src/fs/vhost_user/device.rs @@ -0,0 +1,258 @@ +use crate::device::{SingleFdSignalQueue, VirtioDeviceT}; +use crate::device::{VirtioDevType, VirtioDeviceCommon}; +use crate::mmio::VIRTIO_MMIO_INT_VRING; +use api::device_model::BaoDeviceModel; +use api::error::{Error, Result}; +use api::types::DeviceConfig; +use event_manager::{EventManager, MutEventSubscriber}; +use seccompiler::SeccompAction; +use std::borrow::{Borrow, BorrowMut}; +use std::sync::atomic::{AtomicU8, Ordering}; +use std::sync::{Arc, Mutex}; +use vhost::vhost_user::message::VhostUserProtocolFeatures; +use vhost_user_frontend::{ + Generic as VhostUserCommon, VhostUserConfig, VirtioDevice, + VirtioDeviceType as VhostUserDeviceType, +}; +use virtio_device::{VirtioConfig, VirtioDeviceActions, VirtioDeviceType, VirtioMmioDevice}; +use virtio_queue::{Queue, QueueT}; +use vm_device::bus::MmioAddress; +use vm_device::device_manager::{IoManager, MmioManager}; +use vm_device::MutDeviceMmio; +use vm_memory::GuestMemoryAtomic; +use vmm_sys_util::eventfd::{EventFd, EFD_NONBLOCK}; + +/// Vhost-user file system device. +/// +/// # Attributes +/// +/// * `vhost_user` - Vhost-user generic device. +/// * `virtio` - Virtio virtio device. +/// * `socket_path` - Path to the vhost-user socket. +pub struct VhostUserFs { + pub virtio: VirtioDeviceCommon, + pub vhost_user: Mutex, + pub socket_path: String, +} + +impl VirtioDeviceT for VhostUserFs { + fn new( + config: &DeviceConfig, + device_manager: Arc>, + event_manager: Arc>>>>, + device_model: Arc>, + ) -> Result>> { + // Extract the generic features and queues. + let (common_features, queues) = Self::initialize(&config).unwrap(); + + // Update the configuration space. + let config_space = Self::config_space(&config).unwrap(); + + // Create the vhost-user configuration. + let vu_cfg = VhostUserConfig { + socket: format!( + "{}{}.sock", + config.socket_path.as_ref().unwrap(), + VirtioDevType::from(VirtioDevType::Fs).to_string() + ), + num_queues: queues.len(), + queue_size: queues[0].size(), + }; + + println!( + "Connecting to {} device backend over {} socket..", + VirtioDevType::from(VirtioDevType::Fs).to_string(), + vu_cfg.socket + ); + + // Create the VhostUserCommon vhost-user device. + let vhost_user = VhostUserCommon::new( + vu_cfg, + SeccompAction::Allow, + EventFd::new(EFD_NONBLOCK).unwrap(), + VhostUserDeviceType::Fs, + ) + .map_err(Error::VhostFrontendError)?; + + println!( + "Connected to {} device backend.", + VirtioDevType::from(VirtioDevType::Fs).to_string() + ); + + // Update the device features since we have the vhost-user backend now. + let device_features = Self::device_features(&config).unwrap() + | common_features + | vhost_user.device_features(); + + // Create a VirtioConfig object. + let virtio_cfg = VirtioConfig::new(device_features, queues, config_space); + + // Create the generic device. + let common_device = + VirtioDeviceCommon::new(config, event_manager, device_model, virtio_cfg).unwrap(); + + // Extract the VirtioDeviceCommon MMIO range. + let range = common_device.mmio.range; + + // Create the fs device. + let fs = Arc::new(Mutex::new(VhostUserFs { + vhost_user: Mutex::new(vhost_user), + virtio: common_device, + socket_path: config.socket_path.clone().unwrap(), + })); + + // Register the MMIO device within the device manager with the specified range. + device_manager + .lock() + .unwrap() + .register_mmio(range, fs.clone()) + .unwrap(); + + // Return the fs device. + Ok(fs) + } + + fn device_features(_config: &DeviceConfig) -> Result { + // Here we can leave empty since it is the vhost-user backend responsibility to negotiate the features + // that it supports. + Ok(0) + } + + fn config_space(_config: &DeviceConfig) -> Result> { + Ok(Vec::new()) + } +} + +impl Borrow> for VhostUserFs { + fn borrow(&self) -> &VirtioConfig { + &self.virtio.config + } +} + +impl BorrowMut> for VhostUserFs { + fn borrow_mut(&mut self) -> &mut VirtioConfig { + &mut self.virtio.config + } +} + +impl VirtioDeviceType for VhostUserFs { + fn device_type(&self) -> u32 { + VirtioDevType::Fs as u32 + } +} + +/// Implement the `VirtioDeviceActions` trait to add our custom device actions. +impl VirtioDeviceActions for VhostUserFs { + type E = Error; + + // This method is called after the driver acknowledges all the device features. + // For that reasosn, it is the right place to perform the device initialization. + fn activate(&mut self) -> Result<()> { + // Setup the ioeventfds by calling the generic `prepare_activate` method. + let ioevents = self.virtio.prepare_activate().unwrap(); + + // Create the driver notify object. + let driver_notify = SingleFdSignalQueue { + irqfd: self.virtio.irqfd.try_clone().unwrap(), + interrupt_status: self.virtio.config.interrupt_status.clone(), + }; + + // Format the queues and ioevents into a Vec<(usize, Queue, EventFd)>. + let queues = self + .virtio + .config + .queues + .iter() + .enumerate() + .zip(ioevents) + .map(|((i, queue), ioevent)| (i, queue.clone(), ioevent)) + .collect::>(); + + // Activate the vhost-user device. + self.vhost_user + .lock() + .unwrap() + .activate( + GuestMemoryAtomic::new(self.virtio.mem()), + Arc::new(driver_notify), + queues, + ) + .unwrap(); + + Ok(()) + } + + fn reset(&mut self) -> Result<()> { + // Not implemented for now. + Ok(()) + } + + // This method is called when the driver wants to read information from the device configuration space. + // Since the device configuration space is managed by the device and the device can be implemented in + // different handlers outside of the VMM (vhost or vhost-user) we need to invoke dedicated logic. + fn read_config(&self, offset: usize, data: &mut [u8]) { + self.vhost_user + .lock() + .unwrap() + .read_config(offset as u64, data); + } + + // This method is called when the driver wants to write information to the device configuration space. + // Since the device configuration space is managed by the device and the device can be implemented in + // different handlers outside of the VMM (vhost or vhost-user) we need to invoke dedicated logic. + fn write_config(&mut self, offset: usize, data: &[u8]) { + self.vhost_user + .lock() + .unwrap() + .write_config(offset as u64, data); + } + + // This method is called when the driver finishes the negotiation of the device features + // with the frontend device (selecting page 0). This method is crucial when the device handlers are + // implemented outside of the VMM (vhost or vhost-user) as the frontend device needs to negotiate the + // features with the backend device. Otherwise, the device is not prepared to support, for example, + // multiple queues and configuration space reads and writes. + fn negotiate_driver_features(&mut self) { + self.vhost_user + .lock() + .unwrap() + .negotiate_features( + self.virtio.config.driver_features, + VhostUserProtocolFeatures::empty(), + ) + .unwrap(); + } + + // This method is called when the driver needs to read the interrupt status from the device. + // Since it's the frontend device responsibility to manage the interrupt status, we need to invoke + // dedicated logic to update the interrupt status accordingly (Used Buffer Notification or Configuration Change Notification). + // Note: If the device is implemented in the VMM, the interrupt status can be managed and updated directly by the device. + fn interrupt_status(&self) -> &Arc { + // We assume that all the interrupts are Used Buffer Notifications. + self.virtio + .config + .interrupt_status + .fetch_or(VIRTIO_MMIO_INT_VRING, Ordering::SeqCst); + &self.virtio.config.interrupt_status + } +} + +/// Implement the `VirtioMmioDevice` trait to add VirtIO MMIO support to our device. +impl VirtioMmioDevice for VhostUserFs { + fn queue_notify(&mut self, _val: u32) { + // Do nothing, since the vhost-user backend device is responsible for managing the queue notifications. + // through Ioeventfds. + } +} + +/// Implement the `DeviceMmio` mutable trait to add MMIO support to our device. +/// Otherwise we could not register the device within the device manager. +impl MutDeviceMmio for VhostUserFs { + fn mmio_read(&mut self, _base: MmioAddress, offset: u64, data: &mut [u8]) { + self.read(offset, data); + } + + fn mmio_write(&mut self, _base: MmioAddress, offset: u64, data: &[u8]) { + self.write(offset, data); + } +} diff --git a/src/virtio/src/fs/vhost_user/mod.rs b/src/virtio/src/fs/vhost_user/mod.rs new file mode 100644 index 0000000..5458924 --- /dev/null +++ b/src/virtio/src/fs/vhost_user/mod.rs @@ -0,0 +1 @@ +pub mod device; diff --git a/src/virtio/src/lib.rs b/src/virtio/src/lib.rs new file mode 100644 index 0000000..3229f92 --- /dev/null +++ b/src/virtio/src/lib.rs @@ -0,0 +1,8 @@ +pub mod block; +pub mod device; +pub mod fs; +pub mod mmio; +pub mod net; +pub mod vhost; +pub mod vhost_user; +pub mod vsock; diff --git a/src/virtio/src/mmio.rs b/src/virtio/src/mmio.rs new file mode 100644 index 0000000..41d87bb --- /dev/null +++ b/src/virtio/src/mmio.rs @@ -0,0 +1,68 @@ +use virtio_bindings::virtio_mmio::VIRTIO_MMIO_QUEUE_NOTIFY; +use vm_device::bus::{self, MmioAddress, MmioRange}; + +#[derive(Debug)] +pub enum Error { + Bus(bus::Error), + Overflow, +} + +/// A specialized `Result` type for MMIO operations. +type Result = std::result::Result; + +/// The offset of the queue notify register. +pub const VIRTIO_MMIO_QUEUE_NOTIFY_OFFSET: u64 = VIRTIO_MMIO_QUEUE_NOTIFY as u64; + +/// This bit is set on the device interrupt status when notifying the driver about used +/// queue events (Used Buffer Notification). +pub const VIRTIO_MMIO_INT_VRING: u8 = 0x01; + +/// This bit is set on the device interrupt status when the device configuration has changed +/// (Configuration Change Notification). +pub const VIRTIO_MMIO_INT_CONFIG: u8 = 0x02; + +/// Represents the configuration of a MMIO device. +/// +/// # Attributes +/// +/// * `range` - The MMIO range assigned to the device. +/// * `gsi` - The interrupt assigned to the device. +#[derive(Copy, Clone)] +pub struct MmioConfig { + pub range: MmioRange, + pub gsi: u32, +} + +impl MmioConfig { + /// Creates a new `MmioConfig` object. + /// + /// # Arguments + /// + /// * `base` - The base address of the MMIO range. + /// * `size` - The size of the MMIO range. + /// * `gsi` - The interrupt assigned to the device. + /// + /// # Returns + /// + /// A `Result` containing the `MmioConfig` object. + pub fn new(base: u64, size: u64, gsi: u32) -> Result { + MmioRange::new(MmioAddress(base), size) + .map(|range| MmioConfig { range, gsi }) + .map_err(Error::Bus) + } + + /// Returns the next `MmioConfig` object. + /// + /// # Returns + /// + /// A `Result` containing the next `MmioConfig` object. + pub fn next(&self) -> Result { + let range = self.range; + let next_start = range + .base() + .0 + .checked_add(range.size()) + .ok_or(Error::Overflow)?; + Self::new(next_start, range.size(), self.gsi + 1) + } +} diff --git a/src/virtio/src/net/README.md b/src/virtio/src/net/README.md new file mode 100644 index 0000000..b68a4a5 --- /dev/null +++ b/src/virtio/src/net/README.md @@ -0,0 +1,15 @@ +# VirtIO Net + +## Overview +The VirtIO Net device is a virtual network interface designed for efficient networking within +virtualized environments. It provides a high-performance, low-latency network connection between +virtual machines (VMs), facilitating communication and data transfer over a virtualized network. + +## Purpose +The purpose of the VirtIO Net device is to enable VMs to communicate with each other and with +external networks seamlessly, while minimizing overhead and maximizing performance. By emulating +a network interface that is compatible with the VirtIO standard, the VirtIO Net device allows VMs +to leverage the full capabilities of modern networking technologies within virtualized environments. + +## Requirements +- VirtIO Network support on the Frontend VM (e.g. `CONFIG_VIRTIO_NET` on buildroot) \ No newline at end of file diff --git a/src/virtio/src/net/mod.rs b/src/virtio/src/net/mod.rs new file mode 100644 index 0000000..6804568 --- /dev/null +++ b/src/virtio/src/net/mod.rs @@ -0,0 +1,3 @@ +pub mod utils; +pub mod vhost; +pub mod virtio; diff --git a/src/virtio/src/net/utils.rs b/src/virtio/src/net/utils.rs new file mode 100644 index 0000000..a88a36d --- /dev/null +++ b/src/virtio/src/net/utils.rs @@ -0,0 +1,17 @@ +pub fn mac_address_to_bytes(mac_address: &str) -> Option> { + let parts: Vec<&str> = mac_address.split(':').collect(); + + if parts.len() != 6 { + return None; // MAC address should have 6 segments separated by ':' + } + + let mut bytes = Vec::with_capacity(6); + for part in parts { + match u8::from_str_radix(part, 16) { + Ok(byte) => bytes.push(byte), + Err(_) => return None, // Invalid hexadecimal digit + } + } + + Some(bytes) +} diff --git a/src/virtio/src/net/vhost/device.rs b/src/virtio/src/net/vhost/device.rs new file mode 100644 index 0000000..fa95c32 --- /dev/null +++ b/src/virtio/src/net/vhost/device.rs @@ -0,0 +1,321 @@ +use crate::device::VirtioDeviceT; +use crate::device::{VirtioDevType, VirtioDeviceCommon}; +use crate::mmio::VIRTIO_MMIO_INT_VRING; +use crate::net::utils::mac_address_to_bytes; +use crate::net::virtio::bindings; +use crate::net::virtio::tap::Tap; +use crate::net::virtio::VIRTIO_NET_HDR_SIZE; +use crate::vhost::{VhostKernelCommon, VHOST_FEATURES}; +use api::device_model::BaoDeviceModel; +use api::error::{Error, Result}; +use api::types::DeviceConfig; +use event_manager::{EventManager, MutEventSubscriber}; +use log::error; +use std::borrow::{Borrow, BorrowMut}; +use std::cmp; +use std::sync::atomic::{AtomicU8, Ordering}; +use std::sync::{Arc, Mutex}; +use vhost::net::VhostNet as VhostNetBackend; +use vhost::vhost_kern::net::Net; +use vhost::vhost_kern::VhostKernBackend; +use vhost::{VhostBackend, VringConfigData}; +use vhost_user_frontend::GuestMemoryMmap; +use virtio_bindings::virtio_config::{VIRTIO_F_NOTIFY_ON_EMPTY, VIRTIO_F_RING_RESET}; +use virtio_bindings::virtio_net::VIRTIO_NET_F_MRG_RXBUF; +use virtio_device::{VirtioConfig, VirtioDeviceActions, VirtioDeviceType, VirtioMmioDevice}; +use virtio_queue::{Queue, QueueT}; +use vm_device::bus::MmioAddress; +use vm_device::device_manager::{IoManager, MmioManager}; +use vm_device::MutDeviceMmio; +use vm_memory::GuestAddressSpace; + +const VIRTIO_RING_F_INDIRECT_DESC: u32 = 28; +const VIRTIO_F_RING_EVENT_IDX: u64 = 29; + +/// Vhost net device. +/// +/// # Attributes +/// +/// * `virtio` - Virtio virtio device. +/// * `vhost` - Vhost kernel common device. +/// * `net` - Net device. +/// * `tap_name` - Name of the tap device. +pub struct VhostNet { + pub virtio: VirtioDeviceCommon, + pub vhost: VhostKernelCommon, + pub net: Net>, + pub tap_name: String, +} + +impl VirtioDeviceT for VhostNet { + fn new( + config: &DeviceConfig, + device_manager: Arc>, + event_manager: Arc>>>>, + device_model: Arc>, + ) -> Result>> { + // Extract the generic features and queues. + let (common_features, queues) = Self::initialize(&config).unwrap(); + + // Extract the device features. + let device_features = Self::device_features(&config).unwrap(); + + // Update the configuration space. + let config_space = Self::config_space(&config).unwrap(); + + // Create a VirtioConfig object. + let virtio_cfg = VirtioConfig::new(common_features | device_features, queues, config_space); + + // Create the generic device. + let mut common_device = + VirtioDeviceCommon::new(config, event_manager, device_model, virtio_cfg).unwrap(); + + // Extract the VirtioDeviceCommon MMIO range. + let range = common_device.mmio.range; + + // Create the Net kernel device. + let net_kernel = Net::new(Arc::new(common_device.mem())).unwrap(); + + // Create the net device. + let net = Arc::new(Mutex::new(VhostNet { + virtio: common_device, + vhost: VhostKernelCommon::new(device_features).unwrap(), + net: net_kernel, + tap_name: config.tap_name.clone().unwrap(), + })); + + // Register the MMIO device within the device manager with the specified range. + device_manager + .lock() + .unwrap() + .register_mmio(range, net.clone()) + .unwrap(); + + // Return the net device. + Ok(net) + } + + fn device_features(_config: &DeviceConfig) -> Result { + let features = (1 << VIRTIO_F_RING_EVENT_IDX) + | (1 << VIRTIO_F_NOTIFY_ON_EMPTY) + | (1 << VIRTIO_F_RING_RESET) + | (1 << VIRTIO_RING_F_INDIRECT_DESC) + | (1 << VIRTIO_NET_F_MRG_RXBUF); + + Ok(features | VHOST_FEATURES) + } + + fn config_space(config: &DeviceConfig) -> Result> { + // TODO: Maybe we will need in the future to support setting other fields in the + // configuration space. For now, we only need the mac address. + // Info: https://docs.oasis-open.org/virtio/virtio/v1.2/csd01/virtio-v1.2-csd01.html#x1-2230004 + + // Extract the mac address. + let mut mac_addr = Vec::new(); + if config.mac_addr.is_some() { + mac_addr = mac_address_to_bytes(config.mac_addr.clone().unwrap().as_str()).unwrap(); + } + + // Retrieve the mac address from the device configuration space. + Ok(mac_addr) + } +} + +impl Borrow> for VhostNet { + fn borrow(&self) -> &VirtioConfig { + &self.virtio.config + } +} + +impl BorrowMut> for VhostNet { + fn borrow_mut(&mut self) -> &mut VirtioConfig { + &mut self.virtio.config + } +} + +impl VirtioDeviceType for VhostNet { + fn device_type(&self) -> u32 { + VirtioDevType::Net as u32 + } +} + +/// Implement the `VirtioDeviceActions` trait to add our custom device actions. +impl VirtioDeviceActions for VhostNet { + type E = Error; + + // This method is called after the driver acknowledges all the device features. + // For that reasosn, it is the right place to perform the device initialization. + fn activate(&mut self) -> Result<()> { + // Create the tap device. + let tap = Tap::open_named(self.tap_name.as_str())?; + + // Set offload flags to match the relevant virtio features of the device (for now, + // statically set in the constructor. + tap.set_offload( + bindings::TUN_F_CSUM + | bindings::TUN_F_UFO + | bindings::TUN_F_TSO4 + | bindings::TUN_F_TSO6, + )?; + + // The layout of the header is specified in the standard and is 12 bytes in size. We + // should define this somewhere. + tap.set_vnet_hdr_size(VIRTIO_NET_HDR_SIZE as i32)?; + + // Setup the ioeventfds by calling the generic `prepare_activate` method. + let ioevents = self.virtio.prepare_activate().unwrap(); + + // Format the queues and ioevents into a Vec<(usize, Queue, EventFd)>. + let queues = self + .virtio + .config + .queues + .iter() + .enumerate() + .zip(ioevents) + .map(|((i, queue), ioevent)| (i, queue.clone(), ioevent)) + .collect::>(); + + // Set the current process as the owner of the file descriptor. + self.net.set_owner().unwrap(); + + // Set the device features. + let features = self.net.get_features().unwrap(); + println!("Get Features: {:x}", features); + println!("Set Features: {:x}", self.vhost.features()); + self.net.set_features(self.vhost.features()).unwrap(); + + // Update the memory table. + self.net + .set_mem_table(self.vhost.memory(self.net.mem()).unwrap().as_slice()) + .unwrap(); + + // Set the vring. + let mem = self.net.mem(); + let mem_aux: &GuestMemoryMmap = &mem.memory(); + + for (queue_index, queue, ioeventfd) in queues.iter() { + // Set the vring num. + self.net.set_vring_num(*queue_index, queue.size()).unwrap(); + + let config_data = VringConfigData { + queue_max_size: queue.max_size(), + queue_size: queue.size(), + flags: 0u32, + desc_table_addr: queue.desc_table(), + used_ring_addr: queue.used_ring(), + avail_ring_addr: queue.avail_ring(), + log_addr: None, + }; + + // Set the vring base. + self.net + .set_vring_base( + *queue_index, + queue.avail_idx(mem_aux, Ordering::Acquire).unwrap().0, + ) + .unwrap(); + + // Set the vring address. + self.net.set_vring_addr(*queue_index, &config_data).unwrap(); + + // Set the vring call. + self.net + .set_vring_call(*queue_index, &self.virtio.irqfd.try_clone().unwrap()) + .unwrap(); + + // Set the vring kick. + self.net.set_vring_kick(*queue_index, ioeventfd).unwrap(); + + // Set the backend. + self.net + .set_backend(*queue_index, Some(&tap.tap_file)) + .unwrap(); + } + + Ok(()) + } + + fn reset(&mut self) -> Result<()> { + // Not implemented for now. + Ok(()) + } + + // This method is called when the driver wants to read information from the device configuration space. + // Since the device configuration space is managed by the device and the device can be implemented in + // different handlers outside of the VMM (vhost or vhost-user) we need to invoke dedicated logic. + fn read_config(&self, offset: usize, data: &mut [u8]) { + let config_space = &self.virtio.config.config_space; + let config_len = config_space.len(); + if offset >= config_len { + error!("Failed to read from config space"); + return; + } + + // TODO: Are partial reads ok? + let end = cmp::min(offset.saturating_add(data.len()), config_len); + let read_len = end - offset; + // Cannot fail because the lengths are identical and we do bounds checking beforehand. + data[..read_len].copy_from_slice(&config_space[offset..end]) + } + + // This method is called when the driver wants to write information to the device configuration space. + // Since the device configuration space is managed by the device and the device can be implemented in + // different handlers outside of the VMM (vhost or vhost-user) we need to invoke dedicated logic. + fn write_config(&mut self, offset: usize, data: &[u8]) { + let config_space = &mut self.virtio.config.config_space; + let config_len = config_space.len(); + if offset >= config_len { + error!("Failed to write to config space"); + return; + } + + // TODO: Are partial writes ok? + let end = cmp::min(offset.saturating_add(data.len()), config_len); + let write_len = end - offset; + // Cannot fail because the lengths are identical and we do bounds checking beforehand. + config_space[offset..end].copy_from_slice(&data[..write_len]); + } + + // This method is called when the driver finishes the negotiation of the device features + // with the frontend device (selecting page 0). This method is crucial when the device handlers are + // implemented outside of the VMM (vhost or vhost-user) as the frontend device needs to negotiate the + // features with the backend device. Otherwise, the device is not prepared to support, for example, + // multiple queues and configuration space reads and writes. + fn negotiate_driver_features(&mut self) { + // Do nothing. + } + + // This method is called when the driver needs to read the interrupt status from the device. + // Since it's the frontend device responsibility to manage the interrupt status, we need to invoke + // dedicated logic to update the interrupt status accordingly (Used Buffer Notification or Configuration Change Notification). + // Note: If the device is implemented in the VMM, the interrupt status can be managed and updated directly by the device. + fn interrupt_status(&self) -> &Arc { + // We assume that all the interrupts are Used Buffer Notifications. + self.virtio + .config + .interrupt_status + .fetch_or(VIRTIO_MMIO_INT_VRING, Ordering::SeqCst); + &self.virtio.config.interrupt_status + } +} + +/// Implement the `VirtioMmioDevice` trait to add VirtIO MMIO support to our device. +impl VirtioMmioDevice for VhostNet { + fn queue_notify(&mut self, _val: u32) { + // Do nothing, since the vhost-user backend device is responsible for managing the queue notifications. + // through Ioeventfds. + } +} + +/// Implement the `DeviceMmio` mutable trait to add MMIO support to our device. +/// Otherwise we could not register the device within the device manager. +impl MutDeviceMmio for VhostNet { + fn mmio_read(&mut self, _base: MmioAddress, offset: u64, data: &mut [u8]) { + self.read(offset, data); + } + + fn mmio_write(&mut self, _base: MmioAddress, offset: u64, data: &[u8]) { + self.write(offset, data); + } +} diff --git a/src/virtio/src/net/vhost/mod.rs b/src/virtio/src/net/vhost/mod.rs new file mode 100644 index 0000000..5458924 --- /dev/null +++ b/src/virtio/src/net/vhost/mod.rs @@ -0,0 +1 @@ +pub mod device; diff --git a/src/virtio/src/net/virtio/README.md b/src/virtio/src/net/virtio/README.md new file mode 100644 index 0000000..296c0c5 --- /dev/null +++ b/src/virtio/src/net/virtio/README.md @@ -0,0 +1,155 @@ +# Quick start + +Follow these steps to quickly set up and run a VM with virtio-net capabilities within Bao Hypervisor. + +1. **Prepare the configuration file**: Create a configuration file (e.g. *config-virtio-net.yaml*) specifying +the settings for the virtio-net device. One example of a configuration file could be: + +``` +devices: + # --- VirtIO Common --- + - id: 0 + type: "net" + shmem_addr: 0x50000000 + shmem_size: 0x01000000 + shmem_path: "/dev/baoipc0" + mmio_addr: 0xa003e00 + irq: 47 + data_plane: virtio + # --- Virtio Net Specific --- + tap_name: "tap0" + mac_addr: "52:54:00:12:34:56" + # ----------------------------- +``` + +2. **Create a TAP device**: A TAP (Network TAP) device is a virtual network interface in Linux that operates at the data link layer +(Layer 2) of the OSI model. It simulates an Ethernet network interface and allows user-space applications +to interact with it to transmit and receive Ethernet frames. + +Create a TAP device (e.g. `tap0`) with the mode set to `tap`: +``` +ip tuntap add dev tap0 mode tap +``` + +(Note that the TAP device name should be the same as the tap name in the device model configuration file) + +Assign one IP address (e.g. `192.168.42.14`) to the tap interface and bring it up. +``` +ifconfig tap0 192.168.42.14 up +``` + +3. **Create a bridge**: To enable communication between the TAP device (e.g. `tap0`) and the physical network interface (e.g. `eth0`), +you can create a bridge. A bridge is a virtual network device that connects multiple network interfaces at the +data link layer (Layer 2) and forwards packets between them. + +Create a bridge interface (e.g. `br0`): +``` +brctl addbr br0 +``` + +Assign one IP address (e.g. `192.168.42.13`) to the bridge. +``` +ifconfig br0 192.168.42.13 netmask 255.255.255.0 +``` + +Add the TAP device (`tap0`) and the NIC (`eth0`) to the bridge: +``` +brctl addif br0 eth0 +brctl addif br0 tap0 +``` + +In this context, when the virtio-net driver wants to transmit one packet, the device model forwards the data received +from the frontend to the TAP device, then from the TAP device to the bridge, and finally from the bridge to the physical +NIC driver, and vice versa for returning data from the NIC to the frontend. + +Once the TAP device and the bridge are brought up, you can send and receive network traffic. +You can verify the configuration by running: +``` +ifconfig +``` + +4. **Launch the device model with the virtio-net device**: To launch the device model in the background type: + +``` +nohup bao-virtio --config /PATH/TO/YOUR/config-virtio-net.yaml > /etc/bao-virtio.log 2>&1 & +``` + +In the **Frontend VM**, you can use the command `ethtool` to verify the virtual network interface (e.g. `eth0`) configuration: +``` +ethtool -i eth0 +``` + +(Note: You should visualize the virtio-net driver and the MMIO address that the driver is operating on) + +--- + +## Test cases + +You can validate and test the virtio-net connection by performing the following tests: + +1. Using **ping** command: + +From **any device connected to the network**, type: +``` +ping +``` + +From the **Frontend VM**: +``` +ping +``` + +Additionally, to verify if the Frontend VM has access to the external world, you can simply: +``` +ping 8.8.8.8 +``` + +2. Using **wget** command + +``` +wget http://cdn.kernel.org/pub/linux/kernel/v5.x/linux-5.16.11.tar.xz +``` + +3. Using **iperf3** command to **measure inter-VM or external device connection bandwidth** + +You can use the `iperf3` command to measure the total transmit and receive bandwidth between VMs or between a VM and an external device. + +**Transmit bandwidth**: To measure the transmit bandwidth (guest to host transmit path), +you must run the server on your host and the client on the guest/frontend VM. Note that the host here can be any external device connceted to the network, or even other VM with a network (virtual or physical) interface. + +From the **host**, create the server: +``` +iperf3 -s +``` + +From the **Frontend VM**, create the client: +``` +iperf3 -c +``` + +**Receive bandwidth**: To measure the receive bandwidth (host to guest receive path), +you must run the server on the guest/frontend VM and the client on your host. + +From the **Frontend VM**, create the server: +``` +iperf3 -s +``` + +From the **host**, create the client: +``` +iperf3 -c +``` + +4. Establish a **ssh connection** + +You can establish a `ssh` connection from/to any device connected to the network. + +From **any device connected to the network**, you can connect to the Frontend VM by: +``` +ssh root@ +``` + +Or if you want to establish a conncetion from the **Frontend VM**: +``` +ssh +``` \ No newline at end of file diff --git a/src/virtio/src/net/virtio/bindings.rs b/src/virtio/src/net/virtio/bindings.rs new file mode 100644 index 0000000..be29e2a --- /dev/null +++ b/src/virtio/src/net/virtio/bindings.rs @@ -0,0 +1,262 @@ +// The following are manually copied from crosvm/firecracker. In the latter, they can be found as +// part of the `net_gen` local crate. We should figure out how to proceed going forward (i.e. +// create some bindings of our own, put them in a common crate, etc). + +#![allow(clippy::all)] +#![allow(non_upper_case_globals)] +#![allow(non_camel_case_types)] +#![allow(non_snake_case)] + +pub const TUN_F_CSUM: ::std::os::raw::c_uint = 1; +pub const TUN_F_TSO4: ::std::os::raw::c_uint = 2; +pub const TUN_F_TSO6: ::std::os::raw::c_uint = 4; +pub const TUN_F_UFO: ::std::os::raw::c_uint = 16; + +#[repr(C)] +pub struct __BindgenUnionField(::std::marker::PhantomData); +impl __BindgenUnionField { + #[inline] + pub fn new() -> Self { + __BindgenUnionField(::std::marker::PhantomData) + } + #[inline] + pub unsafe fn as_ref(&self) -> &T { + ::std::mem::transmute(self) + } + #[inline] + pub unsafe fn as_mut(&mut self) -> &mut T { + ::std::mem::transmute(self) + } +} +impl ::std::default::Default for __BindgenUnionField { + #[inline] + fn default() -> Self { + Self::new() + } +} +impl ::std::clone::Clone for __BindgenUnionField { + #[inline] + fn clone(&self) -> Self { + Self::new() + } +} +impl ::std::marker::Copy for __BindgenUnionField {} +impl ::std::fmt::Debug for __BindgenUnionField { + fn fmt(&self, fmt: &mut ::std::fmt::Formatter) -> ::std::fmt::Result { + fmt.write_str("__BindgenUnionField") + } +} + +#[repr(C)] +#[derive(Debug, Default, Copy)] +pub struct ifreq { + pub ifr_ifrn: ifreq__bindgen_ty_1, + pub ifr_ifru: ifreq__bindgen_ty_2, +} + +impl Clone for ifreq { + fn clone(&self) -> Self { + *self + } +} + +#[repr(C)] +#[derive(Debug, Default, Copy)] +pub struct ifreq__bindgen_ty_1 { + pub ifrn_name: __BindgenUnionField<[::std::os::raw::c_uchar; 16usize]>, + pub bindgen_union_field: [u8; 16usize], +} + +impl Clone for ifreq__bindgen_ty_1 { + fn clone(&self) -> Self { + *self + } +} + +#[repr(C)] +#[derive(Debug, Default, Copy)] +pub struct ifreq__bindgen_ty_2 { + pub ifru_addr: __BindgenUnionField, + pub ifru_dstaddr: __BindgenUnionField, + pub ifru_broadaddr: __BindgenUnionField, + pub ifru_netmask: __BindgenUnionField, + pub ifru_hwaddr: __BindgenUnionField, + pub ifru_flags: __BindgenUnionField<::std::os::raw::c_short>, + pub ifru_ivalue: __BindgenUnionField<::std::os::raw::c_int>, + pub ifru_mtu: __BindgenUnionField<::std::os::raw::c_int>, + pub ifru_map: __BindgenUnionField, + pub ifru_slave: __BindgenUnionField<[::std::os::raw::c_char; 16usize]>, + pub ifru_newname: __BindgenUnionField<[::std::os::raw::c_char; 16usize]>, + pub ifru_data: __BindgenUnionField<*mut ::std::os::raw::c_void>, + pub ifru_settings: __BindgenUnionField, + pub bindgen_union_field: [u64; 3usize], +} + +impl Clone for ifreq__bindgen_ty_2 { + fn clone(&self) -> Self { + *self + } +} + +pub type sa_family_t = ::std::os::raw::c_ushort; + +#[repr(C)] +#[derive(Debug, Default, Copy)] +pub struct sockaddr { + pub sa_family: sa_family_t, + pub sa_data: [::std::os::raw::c_char; 14usize], +} + +impl Clone for sockaddr { + fn clone(&self) -> Self { + *self + } +} + +#[repr(C)] +#[derive(Debug, Default, Copy)] +pub struct if_settings { + pub type_: ::std::os::raw::c_uint, + pub size: ::std::os::raw::c_uint, + pub ifs_ifsu: if_settings__bindgen_ty_1, +} + +impl Clone for if_settings { + fn clone(&self) -> Self { + *self + } +} + +#[repr(C)] +#[derive(Debug, Default, Copy)] +pub struct if_settings__bindgen_ty_1 { + pub raw_hdlc: __BindgenUnionField<*mut raw_hdlc_proto>, + pub cisco: __BindgenUnionField<*mut cisco_proto>, + pub fr: __BindgenUnionField<*mut fr_proto>, + pub fr_pvc: __BindgenUnionField<*mut fr_proto_pvc>, + pub fr_pvc_info: __BindgenUnionField<*mut fr_proto_pvc_info>, + pub sync: __BindgenUnionField<*mut sync_serial_settings>, + pub te1: __BindgenUnionField<*mut te1_settings>, + pub bindgen_union_field: u64, +} + +impl Clone for if_settings__bindgen_ty_1 { + fn clone(&self) -> Self { + *self + } +} + +#[repr(C)] +#[derive(Debug, Default, Copy)] +pub struct ifmap { + pub mem_start: ::std::os::raw::c_ulong, + pub mem_end: ::std::os::raw::c_ulong, + pub base_addr: ::std::os::raw::c_ushort, + pub irq: ::std::os::raw::c_uchar, + pub dma: ::std::os::raw::c_uchar, + pub port: ::std::os::raw::c_uchar, +} + +impl Clone for ifmap { + fn clone(&self) -> Self { + *self + } +} + +#[repr(C)] +#[derive(Debug, Default, Copy)] +pub struct raw_hdlc_proto { + pub encoding: ::std::os::raw::c_ushort, + pub parity: ::std::os::raw::c_ushort, +} + +impl Clone for raw_hdlc_proto { + fn clone(&self) -> Self { + *self + } +} + +#[repr(C)] +#[derive(Debug, Default, Copy)] +pub struct cisco_proto { + pub interval: ::std::os::raw::c_uint, + pub timeout: ::std::os::raw::c_uint, +} + +impl Clone for cisco_proto { + fn clone(&self) -> Self { + *self + } +} + +#[repr(C)] +#[derive(Debug, Default, Copy)] +pub struct fr_proto { + pub t391: ::std::os::raw::c_uint, + pub t392: ::std::os::raw::c_uint, + pub n391: ::std::os::raw::c_uint, + pub n392: ::std::os::raw::c_uint, + pub n393: ::std::os::raw::c_uint, + pub lmi: ::std::os::raw::c_ushort, + pub dce: ::std::os::raw::c_ushort, +} + +impl Clone for fr_proto { + fn clone(&self) -> Self { + *self + } +} + +#[repr(C)] +#[derive(Debug, Default, Copy)] +pub struct fr_proto_pvc { + pub dlci: ::std::os::raw::c_uint, +} + +impl Clone for fr_proto_pvc { + fn clone(&self) -> Self { + *self + } +} + +#[repr(C)] +#[derive(Debug, Default, Copy)] +pub struct fr_proto_pvc_info { + pub dlci: ::std::os::raw::c_uint, + pub master: [::std::os::raw::c_char; 16usize], +} + +impl Clone for fr_proto_pvc_info { + fn clone(&self) -> Self { + *self + } +} + +#[repr(C)] +#[derive(Debug, Default, Copy)] +pub struct sync_serial_settings { + pub clock_rate: ::std::os::raw::c_uint, + pub clock_type: ::std::os::raw::c_uint, + pub loopback: ::std::os::raw::c_ushort, +} + +impl Clone for sync_serial_settings { + fn clone(&self) -> Self { + *self + } +} + +#[repr(C)] +#[derive(Debug, Default, Copy)] +pub struct te1_settings { + pub clock_rate: ::std::os::raw::c_uint, + pub clock_type: ::std::os::raw::c_uint, + pub loopback: ::std::os::raw::c_ushort, + pub slot_map: ::std::os::raw::c_uint, +} + +impl Clone for te1_settings { + fn clone(&self) -> Self { + *self + } +} diff --git a/src/virtio/src/net/virtio/device.rs b/src/virtio/src/net/virtio/device.rs new file mode 100644 index 0000000..3cb0481 --- /dev/null +++ b/src/virtio/src/net/virtio/device.rs @@ -0,0 +1,249 @@ +use super::bindings; +use super::queue_handler::QueueHandler; +use super::simple_handler::SimpleHandler; +use super::tap::Tap; +use crate::device::{SingleFdSignalQueue, VirtioDeviceT}; +use crate::device::{VirtioDevType, VirtioDeviceCommon}; +use crate::net::utils::mac_address_to_bytes; +use crate::net::virtio::VIRTIO_NET_HDR_SIZE; +use api::device_model::BaoDeviceModel; +use api::error::{Error, Result}; +use api::types::DeviceConfig; +use event_manager::{EventManager, MutEventSubscriber}; +use log::error; +use std::borrow::{Borrow, BorrowMut}; +use std::cmp; +use std::sync::atomic::AtomicU8; +use std::sync::{Arc, Mutex}; +use virtio_bindings::virtio_config::VIRTIO_F_IN_ORDER; +use virtio_bindings::virtio_net::{ + VIRTIO_NET_F_CSUM, VIRTIO_NET_F_GUEST_CSUM, VIRTIO_NET_F_GUEST_TSO4, VIRTIO_NET_F_GUEST_TSO6, + VIRTIO_NET_F_GUEST_UFO, VIRTIO_NET_F_HOST_TSO4, VIRTIO_NET_F_HOST_TSO6, VIRTIO_NET_F_HOST_UFO, + VIRTIO_NET_F_MAC, +}; +use virtio_device::{VirtioConfig, VirtioDeviceActions, VirtioDeviceType, VirtioMmioDevice}; +use virtio_queue::Queue; +use vm_device::bus::MmioAddress; +use vm_device::device_manager::{IoManager, MmioManager}; +use vm_device::MutDeviceMmio; + +const VIRTIO_F_RING_EVENT_IDX: u64 = 29; + +/// Virtio net device. +/// +/// # Attributes +/// +/// * `common` - Virtio common device. +/// * `tap_name` - Name of the tap device. +pub struct VirtioNet { + pub common: VirtioDeviceCommon, + pub tap_name: String, +} + +impl VirtioDeviceT for VirtioNet { + fn new( + config: &DeviceConfig, + device_manager: Arc>, + event_manager: Arc>>>>, + device_model: Arc>, + ) -> Result>> { + // Extract the generic features and queues. + let (common_features, queues) = Self::initialize(&config).unwrap(); + + // Update the device features. + let device_features = common_features | Self::device_features(&config).unwrap(); + + // Update the configuration space. + let config_space = Self::config_space(&config).unwrap(); + + // Create a VirtioConfig object. + let virtio_cfg = VirtioConfig::new(device_features, queues, config_space); + + // Create the generic device. + let common_device = + VirtioDeviceCommon::new(config, event_manager, device_model, virtio_cfg).unwrap(); + + // Create the net device. + let net = Arc::new(Mutex::new(VirtioNet { + common: common_device, + tap_name: config.tap_name.clone().unwrap(), + })); + + // Register the MMIO device within the device manager with the specified range. + device_manager + .lock() + .unwrap() + .register_mmio(net.clone().lock().unwrap().common.mmio.range, net.clone()) + .unwrap(); + + // Return the net device. + Ok(net) + } + + fn device_features(config: &DeviceConfig) -> Result { + let mut features = (1 << VIRTIO_F_RING_EVENT_IDX) + | (1 << VIRTIO_F_IN_ORDER) + | (1 << VIRTIO_NET_F_CSUM) + | (1 << VIRTIO_NET_F_GUEST_CSUM) + | (1 << VIRTIO_NET_F_GUEST_TSO4) + | (1 << VIRTIO_NET_F_GUEST_TSO6) + | (1 << VIRTIO_NET_F_GUEST_UFO) + | (1 << VIRTIO_NET_F_HOST_TSO4) + | (1 << VIRTIO_NET_F_HOST_TSO6) + | (1 << VIRTIO_NET_F_HOST_UFO); + + // Set the mac address feature if a mac address is provided. + if config.mac_addr.is_some() { + features |= 1 << VIRTIO_NET_F_MAC; + } + + Ok(features) + } + + fn config_space(config: &DeviceConfig) -> Result> { + // TODO: Maybe we will need in the future to support setting other fields in the + // configuration space. For now, we only need the mac address. + // Info: https://docs.oasis-open.org/virtio/virtio/v1.2/csd01/virtio-v1.2-csd01.html#x1-2230004 + + // Extract the mac address. + let mut mac_addr = Vec::new(); + if config.mac_addr.is_some() { + mac_addr = mac_address_to_bytes(config.mac_addr.clone().unwrap().as_str()).unwrap(); + } + + // Retrieve the mac address from the device configuration space. + Ok(mac_addr) + } +} + +impl Borrow> for VirtioNet { + fn borrow(&self) -> &VirtioConfig { + &self.common.config + } +} + +impl BorrowMut> for VirtioNet { + fn borrow_mut(&mut self) -> &mut VirtioConfig { + &mut self.common.config + } +} + +impl VirtioDeviceType for VirtioNet { + fn device_type(&self) -> u32 { + VirtioDevType::Net as u32 + } +} + +/// Implement the `VirtioDeviceActions` trait to add our custom device actions. +impl VirtioDeviceActions for VirtioNet { + type E = Error; + + fn activate(&mut self) -> Result<()> { + // Create the tap device. + let tap = Tap::open_named(self.tap_name.as_str())?; + + // Set offload flags to match the relevant virtio features of the device (for now, + // statically set in the constructor. + tap.set_offload( + bindings::TUN_F_CSUM + | bindings::TUN_F_UFO + | bindings::TUN_F_TSO4 + | bindings::TUN_F_TSO6, + )?; + + // The layout of the header is specified in the standard and is 12 bytes in size. We + // should define this somewhere. + tap.set_vnet_hdr_size(VIRTIO_NET_HDR_SIZE as i32)?; + + // Create the driver notify object. + let driver_notify = SingleFdSignalQueue { + irqfd: self.common.irqfd.try_clone().unwrap(), + interrupt_status: self.common.config.interrupt_status.clone(), + }; + + // Prepare the activation by calling the generic `prepare_activate` method. + let mut ioevents = self.common.prepare_activate()?; + + // Create the inner handler. + let mut queues = self.common.config.queues.clone(); + let rxq = queues.remove(0); + let txq = queues.remove(0); + let inner = SimpleHandler::new(driver_notify, rxq, txq, tap, self.common.mem()); + + // Create the queue handler. + let handler = Arc::new(Mutex::new(QueueHandler { + inner, + rx_ioevent: ioevents.remove(0), + tx_ioevent: ioevents.remove(0), + })); + + // Finalize the activation by calling the generic `finalize_activate` method. + let ret = self.common.finalize_activate(handler); + + Ok(ret.unwrap()) + } + + fn reset(&mut self) -> Result<()> { + // Not implemented for now. + Ok(()) + } + + fn read_config(&self, offset: usize, data: &mut [u8]) { + let config_space = &self.common.config.config_space; + let config_len = config_space.len(); + if offset >= config_len { + error!("Failed to read from config space"); + return; + } + + // TODO: Are partial reads ok? + let end = cmp::min(offset.saturating_add(data.len()), config_len); + let read_len = end - offset; + // Cannot fail because the lengths are identical and we do bounds checking beforehand. + data[..read_len].copy_from_slice(&config_space[offset..end]) + } + + fn write_config(&mut self, offset: usize, data: &[u8]) { + let config_space = &mut self.common.config.config_space; + let config_len = config_space.len(); + if offset >= config_len { + error!("Failed to write to config space"); + return; + } + + // TODO: Are partial writes ok? + let end = cmp::min(offset.saturating_add(data.len()), config_len); + let write_len = end - offset; + // Cannot fail because the lengths are identical and we do bounds checking beforehand. + config_space[offset..end].copy_from_slice(&data[..write_len]); + } + + fn negotiate_driver_features(&mut self) { + // Do nothing here since the features are already negotiated. + } + + fn interrupt_status(&self) -> &Arc { + // Simply return the interrupt status, since the backend (that is inside the VMM) will + // update it. + &self.common.config.interrupt_status + } +} + +/// Implement the `VirtioMmioDevice` trait to add VirtIO MMIO support to our device. +impl VirtioMmioDevice for VirtioNet { + fn queue_notify(&mut self, _val: u32) { + // Do nothing for now. + } +} + +/// Implement the `DeviceMmio` mutable trait to add MMIO support to our device. +/// Otherwise we could not register the device within the device manager. +impl MutDeviceMmio for VirtioNet { + fn mmio_read(&mut self, _base: MmioAddress, offset: u64, data: &mut [u8]) { + self.read(offset, data); + } + + fn mmio_write(&mut self, _base: MmioAddress, offset: u64, data: &[u8]) { + self.write(offset, data); + } +} diff --git a/src/virtio/src/net/virtio/mod.rs b/src/virtio/src/net/virtio/mod.rs new file mode 100644 index 0000000..02f7f4b --- /dev/null +++ b/src/virtio/src/net/virtio/mod.rs @@ -0,0 +1,8 @@ +pub mod bindings; +pub mod device; +mod queue_handler; +mod simple_handler; +pub mod tap; + +// Size of the `virtio_net_hdr` (VirtIO Net header) structure defined by the standard. +pub const VIRTIO_NET_HDR_SIZE: usize = 12; diff --git a/src/virtio/src/net/virtio/queue_handler.rs b/src/virtio/src/net/virtio/queue_handler.rs new file mode 100644 index 0000000..e611f1c --- /dev/null +++ b/src/virtio/src/net/virtio/queue_handler.rs @@ -0,0 +1,96 @@ +use event_manager::{EventOps, Events, MutEventSubscriber}; +use log::error; +use vmm_sys_util::epoll::EventSet; +use vmm_sys_util::eventfd::EventFd; + +use crate::device::SingleFdSignalQueue; + +use super::simple_handler::SimpleHandler; + +const TAPFD_DATA: u32 = 0; +const RX_IOEVENT_DATA: u32 = 1; +const TX_IOEVENT_DATA: u32 = 2; + +/// This object simply combines the more generic `SimpleHandler` with a two concrete queue +/// signalling implementation based on `EventFd`s, and then also implements `MutEventSubscriber` +/// to interact with the event manager. `ioeventfd` is the `EventFd` connected to queue +/// notifications coming from the driver. +/// TODO: Extend this to support multiqueue. +pub struct QueueHandler { + pub inner: SimpleHandler, + pub rx_ioevent: EventFd, + pub tx_ioevent: EventFd, +} + +impl QueueHandler { + // Helper method that receives an error message to be logged and the `ops` handle + // which is used to unregister all events. + fn handle_error>(&self, s: S, ops: &mut EventOps) { + error!("{}", s.as_ref()); + ops.remove(Events::empty(&self.rx_ioevent)) + .expect("Failed to remove rx ioevent"); + ops.remove(Events::empty(&self.tx_ioevent)) + .expect("Failed to remove tx ioevent"); + ops.remove(Events::empty(&self.inner.tap)) + .expect("Failed to remove tap event"); + } +} + +impl MutEventSubscriber for QueueHandler { + fn process(&mut self, events: Events, ops: &mut EventOps) { + // TODO: We can also consider panicking on the errors that cannot be generated + // or influenced. + + if events.event_set() != EventSet::IN { + self.handle_error("Unexpected event_set", ops); + return; + } + + match events.data() { + TAPFD_DATA => { + if let Err(e) = self.inner.process_tap() { + self.handle_error(format!("Process tap error {:?}", e), ops); + } + } + RX_IOEVENT_DATA => { + if self.rx_ioevent.read().is_err() { + self.handle_error("Rx ioevent read", ops); + } else if let Err(e) = self.inner.process_rxq() { + self.handle_error(format!("Process rx error {:?}", e), ops); + } + } + TX_IOEVENT_DATA => { + if self.tx_ioevent.read().is_err() { + self.handle_error("Tx ioevent read", ops); + } + if let Err(e) = self.inner.process_txq() { + self.handle_error(format!("Process tx error {:?}", e), ops); + } + } + _ => self.handle_error("Unexpected data", ops), + } + } + + fn init(&mut self, ops: &mut EventOps) { + ops.add(Events::with_data( + &self.inner.tap, + TAPFD_DATA, + EventSet::IN | EventSet::EDGE_TRIGGERED, + )) + .expect("Unable to add tapfd"); + + ops.add(Events::with_data( + &self.rx_ioevent, + RX_IOEVENT_DATA, + EventSet::IN, + )) + .expect("Unable to add rxfd"); + + ops.add(Events::with_data( + &self.tx_ioevent, + TX_IOEVENT_DATA, + EventSet::IN, + )) + .expect("Unable to add txfd"); + } +} diff --git a/src/virtio/src/net/virtio/simple_handler.rs b/src/virtio/src/net/virtio/simple_handler.rs new file mode 100644 index 0000000..ea3665a --- /dev/null +++ b/src/virtio/src/net/virtio/simple_handler.rs @@ -0,0 +1,193 @@ +use std::cmp; +use std::io::{self, Read, Write}; +use std::result; + +use log::warn; +use virtio_queue::{DescriptorChain, Queue, QueueOwnedT, QueueT}; +use vm_memory::bitmap::AtomicBitmap; +use vm_memory::Bytes; +type GuestMemoryMmap = vm_memory::GuestMemoryMmap; + +use crate::device::SignalUsedQueue; +use crate::net::virtio::tap::Tap; + +// According to the standard: "If the VIRTIO_NET_F_GUEST_TSO4, VIRTIO_NET_F_GUEST_TSO6 or +// VIRTIO_NET_F_GUEST_UFO features are used, the maximum incoming packet will be to 65550 +// bytes long (the maximum size of a TCP or UDP packet, plus the 14 byte ethernet header), +// otherwise 1514 bytes. The 12-byte struct virtio_net_hdr is prepended to this, making for +// 65562 or 1526 bytes." For transmission, the standard states "The header and packet are added +// as one output descriptor to the transmitq, and the device is notified of the new entry". +// We assume the TX frame will not exceed this size either. +const MAX_BUFFER_SIZE: usize = 65562; + +// Prob have to find better names here, but these basically represent the order of the queues. +// If the net device has a single RX/TX pair, then the former has index 0 and the latter 1. When +// the device has multiqueue support, then RX queues have indices 2k, and TX queues 2k+1. +const RXQ_INDEX: u16 = 0; +const TXQ_INDEX: u16 = 1; + +#[derive(Debug)] +pub enum Error { + GuestMemory(vm_memory::GuestMemoryError), + Queue(virtio_queue::Error), + Tap(io::Error), +} + +impl From for Error { + fn from(e: virtio_queue::Error) -> Self { + Error::Queue(e) + } +} + +// A simple handler implementation for a RX/TX queue pair, which does not make assumptions about +// the way queue notification is implemented. The backend is not yet generic (we always assume a +// `Tap` object), but we're looking at improving that going forward. +// TODO: Find a better name. +pub struct SimpleHandler { + pub driver_notify: S, + pub rxq: Queue, + pub rxbuf_current: usize, + pub rxbuf: [u8; MAX_BUFFER_SIZE], + pub txq: Queue, + pub txbuf: [u8; MAX_BUFFER_SIZE], + pub tap: Tap, + pub mem: GuestMemoryMmap, +} + +impl SimpleHandler { + pub fn new(driver_notify: S, rxq: Queue, txq: Queue, tap: Tap, mem: GuestMemoryMmap) -> Self { + SimpleHandler { + driver_notify, + rxq, + rxbuf_current: 0, + rxbuf: [0u8; MAX_BUFFER_SIZE], + txq, + txbuf: [0u8; MAX_BUFFER_SIZE], + tap, + mem, + } + } + + // Have to see how to approach error handling for the `Queue` implementation in particular, + // because many situations are not really recoverable. We should consider reporting them based + // on the metrics/events solution when they appear, and not propagate them further unless + // it's really useful/necessary. + fn write_frame_to_guest(&mut self) -> result::Result { + let num_bytes = self.rxbuf_current; + + let mut chain = match self.rxq.iter(&self.mem)?.next() { + Some(c) => c, + _ => return Ok(false), + }; + + let mut count = 0; + let buf = &mut self.rxbuf[..num_bytes]; + + while let Some(desc) = chain.next() { + let left = buf.len() - count; + + if left == 0 { + break; + } + + let len = cmp::min(left, desc.len() as usize); + chain + .memory() + .write_slice(&buf[count..count + len], desc.addr()) + .map_err(Error::GuestMemory)?; + + count += len; + } + + if count != buf.len() { + // The frame was too large for the chain. + warn!("rx frame too large"); + } + + self.rxq + .add_used(chain.memory(), chain.head_index(), count as u32)?; + + self.rxbuf_current = 0; + + Ok(true) + } + + pub fn process_tap(&mut self) -> result::Result<(), Error> { + loop { + if self.rxbuf_current == 0 { + match self.tap.read(&mut self.rxbuf) { + Ok(n) => self.rxbuf_current = n, + Err(_) => { + // TODO: Do something (logs, metrics, etc.) in response to an error when + // reading from tap. EAGAIN means there's nothing available to read anymore + // (because we open the TAP as non-blocking). + break; + } + } + } + + if !self.write_frame_to_guest()? && !self.rxq.enable_notification(&self.mem)? { + break; + } + } + + if self.rxq.needs_notification(&self.mem)? { + self.driver_notify.signal_used_queue(RXQ_INDEX); + } + + Ok(()) + } + + fn send_frame_from_chain( + &mut self, + chain: &mut DescriptorChain<&GuestMemoryMmap>, + ) -> result::Result { + let mut count = 0; + + while let Some(desc) = chain.next() { + let left = self.txbuf.len() - count; + let len = desc.len() as usize; + + if len > left { + warn!("tx frame too large"); + break; + } + + chain + .memory() + .read_slice(&mut self.txbuf[count..count + len], desc.addr()) + .map_err(Error::GuestMemory)?; + + count += len; + } + + self.tap.write(&self.txbuf[..count]).map_err(Error::Tap)?; + + Ok(count as u32) + } + + pub fn process_txq(&mut self) -> result::Result<(), Error> { + loop { + self.txq.disable_notification(&self.mem)?; + + while let Some(mut chain) = self.txq.iter(&self.mem.clone())?.next() { + self.send_frame_from_chain(&mut chain)?; + + self.txq.add_used(chain.memory(), chain.head_index(), 0)?; + + if self.txq.needs_notification(&self.mem)? { + self.driver_notify.signal_used_queue(TXQ_INDEX); + } + } + + if !self.txq.enable_notification(&self.mem)? { + return Ok(()); + } + } + } + + pub fn process_rxq(&mut self) -> result::Result<(), Error> { + self.rxq.disable_notification(&self.mem)?; + self.process_tap() + } +} diff --git a/src/virtio/src/net/virtio/tap.rs b/src/virtio/src/net/virtio/tap.rs new file mode 100644 index 0000000..89ca0dd --- /dev/null +++ b/src/virtio/src/net/virtio/tap.rs @@ -0,0 +1,182 @@ +// We should add a tap abstraction to rust-vmm as well. Using this one, which is copied from +// Firecracker until then. + +use api::error::{Error, Result}; +use std::fs::File; +use std::io::{Error as IoError, Read, Result as IoResult, Write}; +use std::os::raw::{c_char, c_int, c_uint, c_ulong}; +use std::os::unix::io::{AsRawFd, FromRawFd, RawFd}; + +use vmm_sys_util::ioctl::{ioctl_with_mut_ref, ioctl_with_ref, ioctl_with_val}; +use vmm_sys_util::{ioctl_ioc_nr, ioctl_iow_nr}; + +use super::bindings::ifreq; + +// As defined in the Linux UAPI: +// https://elixir.bootlin.com/linux/v4.17/source/include/uapi/linux/if.h#L33 +const IFACE_NAME_MAX_LEN: usize = 16; + +// Taken from firecracker net_gen/if_tun.rs ... we should see what to do about the net related +// bindings overall for rust-vmm. +const IFF_TAP: ::std::os::raw::c_uint = 2; +const IFF_NO_PI: ::std::os::raw::c_uint = 4096; +const IFF_VNET_HDR: ::std::os::raw::c_uint = 16384; + +const TUNTAP: ::std::os::raw::c_uint = 84; +ioctl_iow_nr!(TUNSETIFF, TUNTAP, 202, ::std::os::raw::c_int); +ioctl_iow_nr!(TUNSETOFFLOAD, TUNTAP, 208, ::std::os::raw::c_uint); +ioctl_iow_nr!(TUNSETVNETHDRSZ, TUNTAP, 216, ::std::os::raw::c_int); + +/// Handle for a network tap interface. +/// +/// For now, this simply wraps the file descriptor for the tap device so methods +/// can run ioctls on the interface. The tap interface fd will be closed when +/// Tap goes out of scope, and the kernel will clean up the interface automatically. +#[derive(Debug)] +pub struct Tap { + pub(crate) tap_file: File, + pub(crate) if_name: [u8; IFACE_NAME_MAX_LEN], +} + +// Returns a byte vector representing the contents of a null terminated C string which +// contains if_name. +fn build_terminated_if_name(if_name: &str) -> Result<[u8; IFACE_NAME_MAX_LEN]> { + // Convert the string slice to bytes, and shadow the variable, + // since we no longer need the &str version. + let if_name_bytes = if_name.as_bytes(); + + if if_name_bytes.len() >= IFACE_NAME_MAX_LEN { + return Err(Error::NetInvalidIfname(if_name.to_string())); + } + + let mut terminated_if_name = [b'\0'; IFACE_NAME_MAX_LEN]; + terminated_if_name[..if_name.len()].copy_from_slice(if_name_bytes); + + Ok(terminated_if_name) +} + +pub struct IfReqBuilder(ifreq); + +impl IfReqBuilder { + #[allow(clippy::new_without_default)] + pub fn new() -> Self { + Self(Default::default()) + } + + pub fn if_name(mut self, if_name: &[u8; IFACE_NAME_MAX_LEN]) -> Self { + // Since we don't call as_mut on the same union field more than once, this block is safe. + let ifrn_name = unsafe { self.0.ifr_ifrn.ifrn_name.as_mut() }; + ifrn_name.copy_from_slice(if_name.as_ref()); + + self + } + + pub(crate) fn flags(mut self, flags: i16) -> Self { + // Since we don't call as_mut on the same union field more than once, this block is safe. + let ifru_flags = unsafe { self.0.ifr_ifru.ifru_flags.as_mut() }; + *ifru_flags = flags; + + self + } + + pub(crate) fn execute(mut self, socket: &F, ioctl: u64) -> Result { + // ioctl is safe. Called with a valid socket fd, and we check the return. + let ret = unsafe { ioctl_with_mut_ref(socket, ioctl, &mut self.0) }; + if ret < 0 { + return Err(Error::IoctlError(IoError::last_os_error())); + } + + Ok(self.0) + } +} + +impl Tap { + /// Create a TUN/TAP device given the interface name. + /// # Arguments + /// + /// * `if_name` - the name of the interface. + pub fn open_named(if_name: &str) -> Result { + let terminated_if_name = build_terminated_if_name(if_name)?; + + let fd = unsafe { + // Open calls are safe because we give a constant null-terminated + // string and verify the result. + libc::open( + b"/dev/net/tun\0".as_ptr() as *const c_char, + libc::O_RDWR | libc::O_NONBLOCK | libc::O_CLOEXEC, + ) + }; + if fd < 0 { + return Err(Error::NetOpenTun(IoError::last_os_error())); + } + // We just checked that the fd is valid. + let tuntap = unsafe { File::from_raw_fd(fd) }; + + let ifreq = IfReqBuilder::new() + .if_name(&terminated_if_name) + .flags((IFF_TAP | IFF_NO_PI | IFF_VNET_HDR) as i16) + .execute(&tuntap, TUNSETIFF())?; + + // Safe since only the name is accessed, and it's cloned out. + Ok(Tap { + tap_file: tuntap, + if_name: unsafe { *ifreq.ifr_ifrn.ifrn_name.as_ref() }, + }) + } + + pub fn if_name_as_str(&self) -> &str { + let len = self + .if_name + .iter() + .position(|x| *x == 0) + .unwrap_or(IFACE_NAME_MAX_LEN); + std::str::from_utf8(&self.if_name[..len]).unwrap_or("") + } + + /// Set the offload flags for the tap interface. + pub fn set_offload(&self, flags: c_uint) -> Result<()> { + // ioctl is safe. Called with a valid tap fd, and we check the return. + let ret = unsafe { ioctl_with_val(&self.tap_file, TUNSETOFFLOAD(), c_ulong::from(flags)) }; + if ret < 0 { + return Err(Error::IoctlError(IoError::last_os_error())); + } + + Ok(()) + } + + /// Set the size of the vnet hdr. + pub fn set_vnet_hdr_size(&self, size: c_int) -> Result<()> { + // ioctl is safe. Called with a valid tap fd, and we check the return. + let ret = unsafe { ioctl_with_ref(&self.tap_file, TUNSETVNETHDRSZ(), &size) }; + if ret < 0 { + return Err(Error::IoctlError(IoError::last_os_error())); + } + + Ok(()) + } +} + +impl Read for Tap { + fn read(&mut self, buf: &mut [u8]) -> IoResult { + self.tap_file.read(buf) + } +} + +impl Write for Tap { + fn write(&mut self, buf: &[u8]) -> IoResult { + self.tap_file.write(buf) + } + + fn flush(&mut self) -> IoResult<()> { + Ok(()) + } +} + +impl AsRawFd for Tap { + fn as_raw_fd(&self) -> RawFd { + self.tap_file.as_raw_fd() + } +} + +// TODO: If we don't end up using an external abstraction for `Tap` interfaces, add unit tests +// based on a mock framework that do not require elevated privileges to run. diff --git a/src/virtio/src/vhost.rs b/src/virtio/src/vhost.rs new file mode 100644 index 0000000..40939fa --- /dev/null +++ b/src/virtio/src/vhost.rs @@ -0,0 +1,43 @@ +use api::error::Result; +use std::sync::Arc; +use vhost::VhostUserMemoryRegionInfo; +use vhost_user_frontend::GuestMemoryMmap; +use vm_memory::{GuestAddressSpace, GuestMemory}; + +pub const VHOST_FEATURES: u64 = 0x13D000000; + +pub struct VhostKernelCommon { + pub features: u64, +} + +impl VhostKernelCommon { + pub fn new(features: u64) -> Result { + Ok(VhostKernelCommon { features }) + } + + pub fn features(&self) -> u64 { + self.features + } + + pub fn memory(&self, mem: &Arc) -> Result> { + let mut regions = Vec::new(); + let mem_region: &GuestMemoryMmap = &mem.memory(); + for region in mem_region.iter() { + let region = match VhostUserMemoryRegionInfo::from_guest_region(region) { + Ok(region) => region, + Err(e) => { + println!("Failed to create memory region: {:?}", e); + panic!("Failed to create memory region: {:?}", e); + } + }; + regions.push(region); + } + + if regions.is_empty() { + println!("No memory regions found"); + panic!("No memory regions found"); + } + + Ok(regions) + } +} diff --git a/src/virtio/src/vhost_user.rs b/src/virtio/src/vhost_user.rs new file mode 100644 index 0000000..8dc597d --- /dev/null +++ b/src/virtio/src/vhost_user.rs @@ -0,0 +1,32 @@ +use super::device::SingleFdSignalQueue; +use std::io::Result as IoResult; +use vhost_user_frontend::{VirtioInterrupt, VirtioInterruptType}; +use vmm_sys_util::eventfd::EventFd; + +impl VirtioInterrupt for SingleFdSignalQueue { + /// Implementation of the trigger method of the VirtioInterrupt trait for BaoInterrupt. + /// + /// # Arguments + /// + /// * `_int_type` - The type of the interrupt (Used Buffer or Configuration Change Notification). + /// + /// # Return + /// + /// * `IoResult<()>` - An IoResult containing Ok(()) on success, or an Error on failure. + fn trigger(&self, _int_type: VirtioInterruptType) -> IoResult<()> { + Ok(()) + } + + /// Implementation of the notifier method of the VirtioInterrupt trait for BaoInterrupt. + /// + /// # Arguments + /// + /// * `_int_type` - The type of the interrupt (Used Buffer or Configuration Change Notification). + /// + /// # Return + /// + /// * `Option` - An Option containing the EventFd associated with the interrupt. + fn notifier(&self, _int_type: VirtioInterruptType) -> Option { + Some(self.irqfd.try_clone().unwrap()) + } +} diff --git a/src/virtio/src/vsock/README.md b/src/virtio/src/vsock/README.md new file mode 100644 index 0000000..bff0201 --- /dev/null +++ b/src/virtio/src/vsock/README.md @@ -0,0 +1,18 @@ +# Virtual Socket + +## Overview +The VirtIO socket (vsock) device is a virtual device that facilitates communication between virtual +machines (VMs). It provides a high-performance, efficient, and secure communication channel for +inter-VM communication within virtualized environments. The application running in the guest communicates over VM sockets i.e over AF_VSOCK sockets. The application running on the host connects to a unix socket on the host i.e communicates over AF_UNIX sockets. +Because the virtual sockets do not rely on +the host’s networking stack at all, it can be used with VMs that have no network interfaces +(Ethernet or TCP/IP stack), reducing the attack surface. + +## Purpose + +The primary purpose of the VirtIO socket device is to enable communication between virtual machines +running on the same host or across different hosts within a virtualized environment. +It allows VMs to exchange data, messages, and other forms of communication securely and efficiently. + +## Requirements +- VirtIO vsockets support on the Frontend VM (e.g. `CONFIG_VIRTIO_VSOCKETS` on buildroot) \ No newline at end of file diff --git a/src/virtio/src/vsock/mod.rs b/src/virtio/src/vsock/mod.rs new file mode 100644 index 0000000..15b5341 --- /dev/null +++ b/src/virtio/src/vsock/mod.rs @@ -0,0 +1,3 @@ +pub mod vhost; +pub mod vhost_user; +pub mod virtio; diff --git a/src/virtio/src/vsock/vhost/README.md b/src/virtio/src/vsock/vhost/README.md new file mode 100644 index 0000000..55ba155 --- /dev/null +++ b/src/virtio/src/vsock/vhost/README.md @@ -0,0 +1,33 @@ +# Vhost vsock + +## Requirements +- Vhost vsock device support on the Backend VM (e.g. `CONFIG_VHOST_VSOCK` on buildroot) + +## Quick start + +Follow these steps to quickly set up and run the vhost vsock device with Bao Hypervisor: + +1. **Prepare Configuration File**: Create a configuration file (*config-virtio-vsock.yaml*) specifying +the settings for the vhost virtual filesystem device. One example of a configuration file could be: + +``` +devices: + # --- Common --- + - id: 0 + type: "vsock" + shmem_addr: 0x50000000 + shmem_size: 0x01000000 + shmem_path: "/dev/baoipc0" + mmio_addr: 0xa003e00 + irq: 47 + data_plane: vhost + # --- Vsock specific --- + guest_cid: 3 + # ----------------------------- +``` + +2. Launch the **device model** with vhost vsock frontend device: + +``` +nohup bao-virtio --config /PATH/TO/YOUR/config-virtio-vsock.yaml > /etc/bao-virtio.log 2>&1 & +``` \ No newline at end of file diff --git a/src/virtio/src/vsock/vhost/device.rs b/src/virtio/src/vsock/vhost/device.rs new file mode 100644 index 0000000..66e143a --- /dev/null +++ b/src/virtio/src/vsock/vhost/device.rs @@ -0,0 +1,287 @@ +use crate::device::VirtioDeviceT; +use crate::device::{VirtioDevType, VirtioDeviceCommon}; +use crate::mmio::VIRTIO_MMIO_INT_VRING; +use crate::vhost::{VhostKernelCommon, VHOST_FEATURES}; +use api::device_model::BaoDeviceModel; +use api::error::{Error, Result}; +use api::types::DeviceConfig; +use event_manager::{EventManager, MutEventSubscriber}; +use log::error; +use std::borrow::{Borrow, BorrowMut}; +use std::cmp; +use std::sync::atomic::{AtomicU8, Ordering}; +use std::sync::{Arc, Mutex}; +use vhost::vhost_kern::vsock::Vsock; +use vhost::vhost_kern::VhostKernBackend; +use vhost::vsock::VhostVsock; +use vhost::{VhostBackend, VringConfigData}; +use vhost_user_frontend::GuestMemoryMmap; +use virtio_device::{VirtioConfig, VirtioDeviceActions, VirtioDeviceType, VirtioMmioDevice}; +use virtio_queue::{Queue, QueueT}; +use vm_device::bus::MmioAddress; +use vm_device::device_manager::{IoManager, MmioManager}; +use vm_device::MutDeviceMmio; +use vm_memory::GuestAddressSpace; + +// Provides a sequenced, reliable, two-way connection-based data transmission path for datagrams of fixed +// maximum length; a consumer is required to read an entire packet with each input system call. +const VIRTIO_VSOCK_F_SEQPACKET: u64 = 1 << 1; + +/// Vhost vsock device. +/// +/// # Attributes +/// +/// * `virtio` - Virtio common device. +/// * `vhost` - Vhost kernel common device. +/// * `vsock` - Vsock device. +/// * `guest_cid` - Guest CID. +pub struct VhostVsockDevice { + pub virtio: VirtioDeviceCommon, + pub vhost: VhostKernelCommon, + pub vsock: Vsock>, + pub guest_cid: u32, +} + +impl VirtioDeviceT for VhostVsockDevice { + fn new( + config: &DeviceConfig, + device_manager: Arc>, + event_manager: Arc>>>>, + device_model: Arc>, + ) -> Result>> { + // Extract the generic features and queues. + let (common_features, queues) = Self::initialize(&config).unwrap(); + + // Extract the device features. + let device_features = Self::device_features(&config).unwrap(); + + // Update the configuration space. + let config_space = Self::config_space(&config).unwrap(); + + // Create a VirtioConfig object. + let virtio_cfg = VirtioConfig::new(common_features | device_features, queues, config_space); + + // Create the generic device. + let mut common_device = + VirtioDeviceCommon::new(config, event_manager, device_model, virtio_cfg).unwrap(); + + // Extract the VirtioDeviceCommon MMIO range. + let range = common_device.mmio.range; + + // Create the Vsock kernel device. + let vsock_kernel = Vsock::new(Arc::new(common_device.mem())).unwrap(); + + // Create the vsock device. + let vsock = Arc::new(Mutex::new(VhostVsockDevice { + virtio: common_device, + vhost: VhostKernelCommon::new(device_features).unwrap(), + vsock: vsock_kernel, + guest_cid: config.guest_cid.unwrap() as u32, + })); + + // Register the MMIO device within the device manager with the specified range. + device_manager + .lock() + .unwrap() + .register_mmio(range, vsock.clone()) + .unwrap(); + + // Return the vosck device. + Ok(vsock) + } + + fn device_features(_config: &DeviceConfig) -> Result { + Ok(VHOST_FEATURES | VIRTIO_VSOCK_F_SEQPACKET) + } + + fn config_space(config: &DeviceConfig) -> Result> { + // Retrieve the guest CID from the device configuration space. + Ok(config.guest_cid.unwrap().to_le_bytes().to_vec()) + } +} + +impl Borrow> for VhostVsockDevice { + fn borrow(&self) -> &VirtioConfig { + &self.virtio.config + } +} + +impl BorrowMut> for VhostVsockDevice { + fn borrow_mut(&mut self) -> &mut VirtioConfig { + &mut self.virtio.config + } +} + +impl VirtioDeviceType for VhostVsockDevice { + fn device_type(&self) -> u32 { + VirtioDevType::Vsock as u32 + } +} + +/// Implement the `VirtioDeviceActions` trait to add our custom device actions. +impl VirtioDeviceActions for VhostVsockDevice { + type E = Error; + + // This method is called after the driver acknowledges all the device features. + // For that reasosn, it is the right place to perform the device initialization. + fn activate(&mut self) -> Result<()> { + // Setup the ioeventfds by calling the generic `prepare_activate` method. + let ioevents = self.virtio.prepare_activate().unwrap(); + + // Format the queues and ioevents into a Vec<(usize, Queue, EventFd)>. + let queues = self + .virtio + .config + .queues + .iter() + .take(2) // The vhost vsock device has only 2 queues (RX/TX), as the Event Queue is not used. + .enumerate() + .zip(ioevents) + .map(|((i, queue), ioevent)| (i, queue.clone(), ioevent)) + .collect::>(); + + // Set the current process as the owner of the file descriptor. + self.vsock.set_owner().unwrap(); + + // Set the device features. + self.vsock.set_features(self.vhost.features()).unwrap(); + + // Update the memory table. + self.vsock + .set_mem_table(self.vhost.memory(self.vsock.mem()).unwrap().as_slice()) + .unwrap(); + + // Set the vring. + let mem = self.vsock.mem(); + let mem_aux: &GuestMemoryMmap = &mem.memory(); + + for (queue_index, queue, ioeventfd) in queues.iter() { + // Set the vring num. + self.vsock + .set_vring_num(*queue_index, queue.size()) + .unwrap(); + + let config_data = VringConfigData { + queue_max_size: queue.max_size(), + queue_size: queue.size(), + flags: 0u32, + desc_table_addr: queue.desc_table(), + used_ring_addr: queue.used_ring(), + avail_ring_addr: queue.avail_ring(), + log_addr: None, + }; + + // Set the vring base. + self.vsock + .set_vring_base( + *queue_index, + queue.avail_idx(mem_aux, Ordering::Acquire).unwrap().0, + ) + .unwrap(); + + // Set the vring address. + self.vsock + .set_vring_addr(*queue_index, &config_data) + .unwrap(); + + // Set the vring call. + self.vsock + .set_vring_call(*queue_index, &self.virtio.irqfd.try_clone().unwrap()) + .unwrap(); + + // Set the vring kick. + self.vsock.set_vring_kick(*queue_index, ioeventfd).unwrap(); + } + + // Set the guest CID. + self.vsock.set_guest_cid(self.guest_cid as u64).unwrap(); + + // Start the vsock device. + self.vsock.start().unwrap(); + + Ok(()) + } + + fn reset(&mut self) -> Result<()> { + // Not implemented for now. + Ok(()) + } + + // This method is called when the driver wants to read information from the device configuration space. + // Since the device configuration space is managed by the device and the device can be implemented in + // different handlers outside of the VMM (vhost or vhost-user) we need to invoke dedicated logic. + fn read_config(&self, offset: usize, data: &mut [u8]) { + let config_space = &self.virtio.config.config_space; + let config_len = config_space.len(); + if offset >= config_len { + error!("Failed to read from config space"); + return; + } + + // TODO: Are partial reads ok? + let end = cmp::min(offset.saturating_add(data.len()), config_len); + let read_len = end - offset; + // Cannot fail because the lengths are identical and we do bounds checking beforehand. + data[..read_len].copy_from_slice(&config_space[offset..end]) + } + + // This method is called when the driver wants to write information to the device configuration space. + // Since the device configuration space is managed by the device and the device can be implemented in + // different handlers outside of the VMM (vhost or vhost-user) we need to invoke dedicated logic. + fn write_config(&mut self, offset: usize, data: &[u8]) { + let config_space = &mut self.virtio.config.config_space; + let config_len = config_space.len(); + if offset >= config_len { + error!("Failed to write to config space"); + return; + } + + // TODO: Are partial writes ok? + let end = cmp::min(offset.saturating_add(data.len()), config_len); + let write_len = end - offset; + // Cannot fail because the lengths are identical and we do bounds checking beforehand. + config_space[offset..end].copy_from_slice(&data[..write_len]); + } + + // This method is called when the driver finishes the negotiation of the device features + // with the frontend device (selecting page 0). This method is crucial when the device handlers are + // implemented outside of the VMM (vhost or vhost-user) as the frontend device needs to negotiate the + // features with the backend device. Otherwise, the device is not prepared to support, for example, + // multiple queues and configuration space reads and writes. + fn negotiate_driver_features(&mut self) { + // Do nothing. + } + + // This method is called when the driver needs to read the interrupt status from the device. + // Since it's the frontend device responsibility to manage the interrupt status, we need to invoke + // dedicated logic to update the interrupt status accordingly (Used Buffer Notification or Configuration Change Notification). + // Note: If the device is implemented in the VMM, the interrupt status can be managed and updated directly by the device. + fn interrupt_status(&self) -> &Arc { + // We assume that all the interrupts are Used Buffer Notifications. + self.virtio + .config + .interrupt_status + .fetch_or(VIRTIO_MMIO_INT_VRING, Ordering::SeqCst); + &self.virtio.config.interrupt_status + } +} + +/// Implement the `VirtioMmioDevice` trait to add VirtIO MMIO support to our device. +impl VirtioMmioDevice for VhostVsockDevice { + fn queue_notify(&mut self, _val: u32) { + // Do nothing, since the vhost-kernel backend device is responsible for managing the queue notifications + // through Ioeventfds. + } +} + +/// Implement the `DeviceMmio` mutable trait to add MMIO support to our device. +/// Otherwise we could not register the device within the device manager. +impl MutDeviceMmio for VhostVsockDevice { + fn mmio_read(&mut self, _base: MmioAddress, offset: u64, data: &mut [u8]) { + self.read(offset, data); + } + + fn mmio_write(&mut self, _base: MmioAddress, offset: u64, data: &[u8]) { + self.write(offset, data); + } +} diff --git a/src/virtio/src/vsock/vhost/mod.rs b/src/virtio/src/vsock/vhost/mod.rs new file mode 100644 index 0000000..5458924 --- /dev/null +++ b/src/virtio/src/vsock/vhost/mod.rs @@ -0,0 +1 @@ +pub mod device; diff --git a/src/virtio/src/vsock/vhost_user/device.rs b/src/virtio/src/vsock/vhost_user/device.rs new file mode 100644 index 0000000..20f4173 --- /dev/null +++ b/src/virtio/src/vsock/vhost_user/device.rs @@ -0,0 +1,259 @@ +use crate::device::{SingleFdSignalQueue, VirtioDeviceT}; +use crate::device::{VirtioDevType, VirtioDeviceCommon}; +use crate::mmio::VIRTIO_MMIO_INT_VRING; +use api::device_model::BaoDeviceModel; +use api::error::{Error, Result}; +use api::types::DeviceConfig; +use event_manager::{EventManager, MutEventSubscriber}; +use seccompiler::SeccompAction; +use std::borrow::{Borrow, BorrowMut}; +use std::sync::atomic::{AtomicU8, Ordering}; +use std::sync::{Arc, Mutex}; +use vhost::vhost_user::message::VhostUserProtocolFeatures; +use vhost_user_frontend::{ + Generic, VhostUserConfig, VirtioDevice, VirtioDeviceType as VhostUserDeviceType, +}; +use virtio_device::{VirtioConfig, VirtioDeviceActions, VirtioDeviceType, VirtioMmioDevice}; +use virtio_queue::{Queue, QueueT}; +use vm_device::bus::MmioAddress; +use vm_device::device_manager::{IoManager, MmioManager}; +use vm_device::MutDeviceMmio; +use vm_memory::GuestMemoryAtomic; +use vmm_sys_util::eventfd::{EventFd, EFD_NONBLOCK}; + +/// Vhost-user vsock device. +/// +/// # Attributes +/// +/// * `virtio` - Virtio virtio device. +/// * `vhost_user` - Vhost-user generic device. +/// * `socket_path` - Path to the vhost-user socket. +pub struct VhostUserVsock { + pub virtio: VirtioDeviceCommon, + pub vhost_user: Mutex, + pub socket_path: String, +} + +impl VirtioDeviceT for VhostUserVsock { + fn new( + config: &DeviceConfig, + device_manager: Arc>, + event_manager: Arc>>>>, + device_model: Arc>, + ) -> Result>> { + // Extract the generic features and queues. + let (common_features, queues) = Self::initialize(&config).unwrap(); + + // Update the configuration space. + let config_space = Self::config_space(&config).unwrap(); + + // Create the vhost-user configuration. + let vu_cfg = VhostUserConfig { + socket: format!( + "{}{}.sock", + config.socket_path.as_ref().unwrap(), + VirtioDevType::from(VirtioDevType::Vsock).to_string() + ), + num_queues: 1, // Currently, multiple queues are not supported by the vhost-user backend (feature `VhostUserProtocolFeatures::MQ` not negotiated). + queue_size: queues[0].size(), + }; + + println!( + "Connecting to {} device backend over {} socket..", + VirtioDevType::from(VirtioDevType::Vsock).to_string(), + vu_cfg.socket + ); + + // Create the Generic vhost-user device. + let vhost_user = Generic::new( + vu_cfg, + SeccompAction::Allow, + EventFd::new(EFD_NONBLOCK).unwrap(), + VhostUserDeviceType::Vsock, + ) + .map_err(Error::VhostFrontendError)?; + + println!( + "Connected to {} device backend.", + VirtioDevType::from(VirtioDevType::Vsock).to_string() + ); + + // Update the device features since we have the vhost-user backend now. + let device_features = Self::device_features(&config).unwrap() + | common_features + | vhost_user.device_features(); + + // Create a VirtioConfig object. + let virtio_cfg = VirtioConfig::new(device_features, queues, config_space); + + // Create the generic device. + let common_device = + VirtioDeviceCommon::new(config, event_manager, device_model, virtio_cfg).unwrap(); + + // Extract the VirtioDeviceCommon MMIO range. + let range = common_device.mmio.range; + + // Create the vsock device. + let vsock = Arc::new(Mutex::new(VhostUserVsock { + vhost_user: Mutex::new(vhost_user), + virtio: common_device, + socket_path: config.socket_path.clone().unwrap(), + })); + + // Register the MMIO device within the device manager with the specified range. + device_manager + .lock() + .unwrap() + .register_mmio(range, vsock.clone()) + .unwrap(); + + // Return the vsock device. + Ok(vsock) + } + + fn device_features(_config: &DeviceConfig) -> Result { + // Here we can leave empty since it is the vhost-user backend responsibility to negotiate the features + // that it supports. + Ok(0) + } + + fn config_space(config: &DeviceConfig) -> Result> { + // Retrieve the guest CID from the device configuration space. + Ok(config.guest_cid.unwrap().to_le_bytes().to_vec()) + } +} + +impl Borrow> for VhostUserVsock { + fn borrow(&self) -> &VirtioConfig { + &self.virtio.config + } +} + +impl BorrowMut> for VhostUserVsock { + fn borrow_mut(&mut self) -> &mut VirtioConfig { + &mut self.virtio.config + } +} + +impl VirtioDeviceType for VhostUserVsock { + fn device_type(&self) -> u32 { + VirtioDevType::Vsock as u32 + } +} + +/// Implement the `VirtioDeviceActions` trait to add our custom device actions. +impl VirtioDeviceActions for VhostUserVsock { + type E = Error; + + // This method is called after the driver acknowledges all the device features. + // For that reasosn, it is the right place to perform the device initialization. + fn activate(&mut self) -> Result<()> { + // Setup the ioeventfds by calling the generic `prepare_activate` method. + let ioevents = self.virtio.prepare_activate().unwrap(); + + // Create the driver notify object. + let driver_notify = SingleFdSignalQueue { + irqfd: self.virtio.irqfd.try_clone().unwrap(), + interrupt_status: self.virtio.config.interrupt_status.clone(), + }; + + // Format the queues and ioevents into a Vec<(usize, Queue, EventFd)>. + let queues = self + .virtio + .config + .queues + .iter() + .enumerate() + .take(1) // Currently, multiple queues are not supported by the vhost-user backend (feature `VhostUserProtocolFeatures::MQ` not negotiated). + .zip(ioevents) + .map(|((i, queue), ioevent)| (i, queue.clone(), ioevent)) + .collect::>(); + + // Activate the vhost-user device. + self.vhost_user + .lock() + .unwrap() + .activate( + GuestMemoryAtomic::new(self.virtio.mem()), + Arc::new(driver_notify), + queues, + ) + .unwrap(); + + Ok(()) + } + + fn reset(&mut self) -> Result<()> { + // Not implemented for now. + Ok(()) + } + + // This method is called when the driver wants to read information from the device configuration space. + // Since the device configuration space is managed by the device and the device can be implemented in + // different handlers outside of the VMM (vhost or vhost-user) we need to invoke dedicated logic. + fn read_config(&self, offset: usize, data: &mut [u8]) { + self.vhost_user + .lock() + .unwrap() + .read_config(offset as u64, data); + } + + // This method is called when the driver wants to write information to the device configuration space. + // Since the device configuration space is managed by the device and the device can be implemented in + // different handlers outside of the VMM (vhost or vhost-user) we need to invoke dedicated logic. + fn write_config(&mut self, offset: usize, data: &[u8]) { + self.vhost_user + .lock() + .unwrap() + .write_config(offset as u64, data); + } + + // This method is called when the driver finishes the negotiation of the device features + // with the frontend device (selecting page 0). This method is crucial when the device handlers are + // implemented outside of the VMM (vhost or vhost-user) as the frontend device needs to negotiate the + // features with the backend device. Otherwise, the device is not prepared to support, for example, + // multiple queues and configuration space reads and writes. + fn negotiate_driver_features(&mut self) { + self.vhost_user + .lock() + .unwrap() + .negotiate_features( + self.virtio.config.driver_features, + VhostUserProtocolFeatures::empty(), + ) + .unwrap(); + } + + // This method is called when the driver needs to read the interrupt status from the device. + // Since it's the frontend device responsibility to manage the interrupt status, we need to invoke + // dedicated logic to update the interrupt status accordingly (Used Buffer Notification or Configuration Change Notification). + // Note: If the device is implemented in the VMM, the interrupt status can be managed and updated directly by the device. + fn interrupt_status(&self) -> &Arc { + // We assume that all the interrupts are Used Buffer Notifications. + self.virtio + .config + .interrupt_status + .fetch_or(VIRTIO_MMIO_INT_VRING, Ordering::SeqCst); + &self.virtio.config.interrupt_status + } +} + +/// Implement the `VirtioMmioDevice` trait to add VirtIO MMIO support to our device. +impl VirtioMmioDevice for VhostUserVsock { + fn queue_notify(&mut self, _val: u32) { + // Do nothing, since the vhost-user backend device is responsible for managing the queue notifications. + // through Ioeventfds. + } +} + +/// Implement the `DeviceMmio` mutable trait to add MMIO support to our device. +/// Otherwise we could not register the device within the device manager. +impl MutDeviceMmio for VhostUserVsock { + fn mmio_read(&mut self, _base: MmioAddress, offset: u64, data: &mut [u8]) { + self.read(offset, data); + } + + fn mmio_write(&mut self, _base: MmioAddress, offset: u64, data: &[u8]) { + self.write(offset, data); + } +} diff --git a/src/virtio/src/vsock/vhost_user/mod.rs b/src/virtio/src/vsock/vhost_user/mod.rs new file mode 100644 index 0000000..5458924 --- /dev/null +++ b/src/virtio/src/vsock/vhost_user/mod.rs @@ -0,0 +1 @@ +pub mod device; diff --git a/src/virtio/src/vsock/virtio/device.rs b/src/virtio/src/vsock/virtio/device.rs new file mode 100644 index 0000000..df9af10 --- /dev/null +++ b/src/virtio/src/vsock/virtio/device.rs @@ -0,0 +1,197 @@ +use super::packet_handler::VsockPacketHandler; +use super::queue_handler::QueueHandler; +use crate::device::{SingleFdSignalQueue, VirtioDeviceT}; +use crate::device::{VirtioDevType, VirtioDeviceCommon}; +use api::device_model::BaoDeviceModel; +use api::error::{Error, Result}; +use api::types::DeviceConfig; +use event_manager::{EventManager, MutEventSubscriber}; +use log::error; +use std::borrow::{Borrow, BorrowMut}; +use std::cmp; +use std::sync::atomic::AtomicU8; +use std::sync::{Arc, Mutex}; +use virtio_device::{VirtioConfig, VirtioDeviceActions, VirtioDeviceType, VirtioMmioDevice}; +use virtio_queue::Queue; +use vm_device::bus::MmioAddress; +use vm_device::device_manager::{IoManager, MmioManager}; +use vm_device::MutDeviceMmio; + +/// Virtio vsock device. +/// +/// # Attributes +/// +/// * `common` - Virtio common device. +/// * `guest_cid` - The guest CID. +pub struct VirtioVsock { + pub common: VirtioDeviceCommon, + pub guest_cid: u64, +} + +impl VirtioDeviceT for VirtioVsock { + fn new( + config: &DeviceConfig, + device_manager: Arc>, + event_manager: Arc>>>>, + device_model: Arc>, + ) -> Result>> { + // Extract the generic features and queues. + let (common_features, queues) = Self::initialize(&config).unwrap(); + + // Update the device features. + let device_features = common_features | Self::device_features(&config).unwrap(); + + // Update the configuration space. + let config_space = Self::config_space(&config).unwrap(); + + // Create a VirtioConfig object. + let virtio_cfg = VirtioConfig::new(device_features, queues, config_space); + + // Create the generic device. + let common_device = + VirtioDeviceCommon::new(config, event_manager, device_model, virtio_cfg).unwrap(); + + // Create the vsock device. + let vsock = Arc::new(Mutex::new(VirtioVsock { + common: common_device, + guest_cid: config.guest_cid.unwrap(), + })); + + // Register the MMIO device within the device manager with the specified range. + device_manager + .lock() + .unwrap() + .register_mmio( + vsock.clone().lock().unwrap().common.mmio.range, + vsock.clone(), + ) + .unwrap(); + + // Return the vsock device. + Ok(vsock) + } + + fn device_features(_config: &DeviceConfig) -> Result { + Ok(0) + } + + fn config_space(_config: &DeviceConfig) -> Result> { + Ok(Vec::new()) + } +} + +impl Borrow> for VirtioVsock { + fn borrow(&self) -> &VirtioConfig { + &self.common.config + } +} + +impl BorrowMut> for VirtioVsock { + fn borrow_mut(&mut self) -> &mut VirtioConfig { + &mut self.common.config + } +} + +impl VirtioDeviceType for VirtioVsock { + fn device_type(&self) -> u32 { + VirtioDevType::Vsock as u32 + } +} + +/// Implement the `VirtioDeviceActions` trait to add our custom device actions. +impl VirtioDeviceActions for VirtioVsock { + type E = Error; + + fn activate(&mut self) -> Result<()> { + // Create the driver notify object. + let driver_notify = SingleFdSignalQueue { + irqfd: self.common.irqfd.try_clone().unwrap(), + interrupt_status: self.common.config.interrupt_status.clone(), + }; + + // Prepare the activation by calling the generic `prepare_activate` method. + let ioevents = self.common.prepare_activate().unwrap(); + + // Create the inner handler. + let inner = VsockPacketHandler { + driver_notify, + mem: self.common.mem(), + queues: self.common.config.queues.clone(), + }; + + // Create the queue handler. + let handler = Arc::new(Mutex::new(QueueHandler { + inner, + ioeventfd: ioevents, + })); + + // Finalize the activation by calling the generic `finalize_activate` method. + let ret = self.common.finalize_activate(handler); + + Ok(ret.unwrap()) + } + + fn reset(&mut self) -> Result<()> { + // Not implemented for now. + Ok(()) + } + + fn read_config(&self, offset: usize, data: &mut [u8]) { + let config_space = &self.common.config.config_space; + let config_len = config_space.len(); + if offset >= config_len { + error!("Failed to read from config space"); + return; + } + + // TODO: Are partial reads ok? + let end = cmp::min(offset.saturating_add(data.len()), config_len); + let read_len = end - offset; + // Cannot fail because the lengths are identical and we do bounds checking beforehand. + data[..read_len].copy_from_slice(&config_space[offset..end]) + } + + fn write_config(&mut self, offset: usize, data: &[u8]) { + let config_space = &mut self.common.config.config_space; + let config_len = config_space.len(); + if offset >= config_len { + error!("Failed to write to config space"); + return; + } + + // TODO: Are partial writes ok? + let end = cmp::min(offset.saturating_add(data.len()), config_len); + let write_len = end - offset; + // Cannot fail because the lengths are identical and we do bounds checking beforehand. + config_space[offset..end].copy_from_slice(&data[..write_len]); + } + + fn negotiate_driver_features(&mut self) { + // Do nothing here since the features are already negotiated. + } + + fn interrupt_status(&self) -> &Arc { + // Simply return the interrupt status, since the backend (that is inside the VMM) will + // update it. + &self.common.config.interrupt_status + } +} + +/// Implement the `VirtioMmioDevice` trait to add VirtIO MMIO support to our device. +impl VirtioMmioDevice for VirtioVsock { + fn queue_notify(&mut self, _val: u32) { + // Do nothing for now. + } +} + +/// Implement the `DeviceMmio` mutable trait to add MMIO support to our device. +/// Otherwise we could not register the device within the device manager. +impl MutDeviceMmio for VirtioVsock { + fn mmio_read(&mut self, _base: MmioAddress, offset: u64, data: &mut [u8]) { + self.read(offset, data); + } + + fn mmio_write(&mut self, _base: MmioAddress, offset: u64, data: &[u8]) { + self.write(offset, data); + } +} diff --git a/src/virtio/src/vsock/virtio/mod.rs b/src/virtio/src/vsock/virtio/mod.rs new file mode 100644 index 0000000..71034ac --- /dev/null +++ b/src/virtio/src/vsock/virtio/mod.rs @@ -0,0 +1,3 @@ +pub mod device; +pub mod packet_handler; +pub mod queue_handler; diff --git a/src/virtio/src/vsock/virtio/packet_handler.rs b/src/virtio/src/vsock/virtio/packet_handler.rs new file mode 100644 index 0000000..a80d491 --- /dev/null +++ b/src/virtio/src/vsock/virtio/packet_handler.rs @@ -0,0 +1,123 @@ +use crate::device::SignalUsedQueue; +use api::error::{Error, Result}; +use virtio_queue::{DescriptorChain, Queue, QueueOwnedT, QueueT}; +use virtio_vsock::packet::VsockPacket; +use vm_memory::bitmap::AtomicBitmap; + +type GuestMemoryMmap = vm_memory::GuestMemoryMmap; + +const MAX_PKT_BUF_SIZE: u32 = 64 * 1024; + +const RX_VIRTQ: usize = 0; +const TX_VIRTQ: usize = 1; + +const OP_RW: u16 = 5; + +pub struct VsockPacketHandler { + pub driver_notify: S, + pub mem: GuestMemoryMmap, + pub queues: Vec, +} + +impl VsockPacketHandler +where + S: SignalUsedQueue, +{ + /// Process a chain. + fn process_chain( + &mut self, + mut chain: DescriptorChain<&GuestMemoryMmap>, + queue_index: usize, + ) -> Result<()> { + let vsock_packet; + match queue_index { + RX_VIRTQ => { + vsock_packet = + VsockPacket::from_rx_virtq_chain(&self.mem, &mut chain, MAX_PKT_BUF_SIZE) + .unwrap(); + /* + // Write data to the packet, using the setters. + vsock_packet.set_src_cid(SRC_CID) + .set_dst_cid(DST_CID) + .set_src_port(SRC_PORT) + .set_dst_port(DST_PORT) + .set_type(TYPE_STREAM) + .set_buf_alloc(BUF_ALLOC) + .set_fwd_cnt(FWD_CNT); + // In this example, we are sending a RW packet. + vsock_packet.data_slice() + .unwrap() + .write_slice(&[1u8; LEN as usize], 0); + vsock_packet.set_op(OP_RW).set_len(LEN); + vsock_packet.header_slice().len() as u32 + LEN + */ + } + TX_VIRTQ => { + vsock_packet = + VsockPacket::from_rx_virtq_chain(&self.mem, &mut chain, MAX_PKT_BUF_SIZE) + .unwrap(); + if vsock_packet.op() == OP_RW { + // Send the packet payload to the backend. + } + } + _ => { + println!("Invalid queue index: {}", queue_index); + return Err(Error::DeviceNotFound); + } + } + + // Add the used descriptor to the queue. + self.queues[queue_index] + .add_used(chain.memory(), chain.head_index(), vsock_packet.len()) + .unwrap(); + + // Signal the driver, if needed. + if self.queues[queue_index] + .needs_notification(chain.memory()) + .unwrap() + { + self.driver_notify.signal_used_queue(0); + } + + Ok(()) + } + + /// Process the queue. + /// + /// # Arguments + /// + /// * `queue_index` - The index of the queue to process. + /// + /// # Returns + /// + /// * `()` - Ok if the queue was processed successfully. + pub fn process_queue(&mut self, queue_index: usize) -> Result<()> { + // To see why this is done in a loop, please look at the `Queue::enable_notification` + // comments in `virtio_queue`. + loop { + // Disable the notifications. + self.queues[queue_index] + .disable_notification(&self.mem) + .unwrap(); + + // Process the queue. + while let Some(chain) = self.queues[queue_index] + .iter(&self.mem.clone()) + .unwrap() + .next() + { + self.process_chain(chain, queue_index)?; + } + + // Enable the notifications. + if !self.queues[queue_index] + .enable_notification(&self.mem) + .unwrap() + { + break; + } + } + + Ok(()) + } +} diff --git a/src/virtio/src/vsock/virtio/queue_handler.rs b/src/virtio/src/vsock/virtio/queue_handler.rs new file mode 100644 index 0000000..78ac052 --- /dev/null +++ b/src/virtio/src/vsock/virtio/queue_handler.rs @@ -0,0 +1,46 @@ +use super::packet_handler::VsockPacketHandler; +use crate::device::SingleFdSignalQueue; +use event_manager::{EventOps, Events, MutEventSubscriber}; +use vmm_sys_util::epoll::EventSet; +use vmm_sys_util::eventfd::EventFd; + +// This object simply combines the more generic `VsockPacketHandler` with a concrete queue +// signalling implementation based on `EventFd`s, and then also implements `MutEventSubscriber` +// to interact with the event manager. `ioeventfd` is the `EventFd` connected to queue +// notifications coming from the driver. +pub(crate) struct QueueHandler { + pub inner: VsockPacketHandler, + pub ioeventfd: Vec, +} + +/// Implement the `MutEventSubscriber` trait for `QueueHandler` to handle the dispatched +/// events (Ioeventfds) from the event manager. +impl MutEventSubscriber for QueueHandler { + fn process(&mut self, events: Events, ops: &mut EventOps) { + let mut error = true; + + // TODO: Have a look at any potential performance impact caused by these conditionals + // just to be sure. + if events.event_set() != EventSet::IN { + println!("unexpected event_set"); + } else if events.data() as usize >= self.ioeventfd.len() { + println!("unexpected events data {}", events.data()); + } else if let Err(e) = self.inner.process_queue(events.data() as usize) { + println!("error processing block queue {:?}", e); + } else { + error = false; + } + + if error { + ops.remove(events) + .expect("Failed to remove fd from event handling loop"); + } + } + + fn init(&mut self, ops: &mut EventOps) { + for (index, ioeventfd) in self.ioeventfd.drain(..).enumerate() { + ops.add(Events::with_data(&ioeventfd, index as u32, EventSet::IN)) + .expect("Failed to init block queue handler"); + } + } +} diff --git a/src/vmm/.gitignore b/src/vmm/.gitignore new file mode 100644 index 0000000..ea8c4bf --- /dev/null +++ b/src/vmm/.gitignore @@ -0,0 +1 @@ +/target diff --git a/src/vmm/Cargo.lock b/src/vmm/Cargo.lock new file mode 100644 index 0000000..52445d7 --- /dev/null +++ b/src/vmm/Cargo.lock @@ -0,0 +1,447 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 3 + +[[package]] +name = "api" +version = "0.1.0" +dependencies = [ + "clap", + "event-manager", + "lazy_static", + "libc", + "serde", + "serde_yaml", + "thiserror", + "vhost-user-frontend", + "vmm-sys-util", +] + +[[package]] +name = "arc-swap" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bddcadddf5e9015d310179a59bb28c4d4b9920ad0f11e8e14dbadf654890c9a6" + +[[package]] +name = "atty" +version = "0.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8" +dependencies = [ + "hermit-abi", + "libc", + "winapi", +] + +[[package]] +name = "autocfg" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" + +[[package]] +name = "bitflags" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" + +[[package]] +name = "bitflags" +version = "2.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed570934406eb16438a4e976b1b4500774099c13b8cb96eec99f620f05090ddf" + +[[package]] +name = "clap" +version = "3.2.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ea181bf566f71cb9a5d17a59e1871af638180a18fb0035c92ae62b705207123" +dependencies = [ + "atty", + "bitflags 1.3.2", + "clap_lex", + "indexmap", + "strsim", + "termcolor", + "textwrap", +] + +[[package]] +name = "clap_lex" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2850f2f5a82cbf437dd5af4d49848fbdfc27c157c3d010345776f952765261c5" +dependencies = [ + "os_str_bytes", +] + +[[package]] +name = "devices" +version = "0.1.0" +dependencies = [ + "api", + "event-manager", + "libc", + "log", + "seccompiler", + "vhost", + "vhost-user-frontend", + "virtio-bindings 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)", + "virtio-blk", + "virtio-device", + "virtio-queue", + "vm-device", + "vm-memory", + "vmm-sys-util", +] + +[[package]] +name = "epoll" +version = "4.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "74351c3392ea1ff6cd2628e0042d268ac2371cb613252ff383b6dfa50d22fa79" +dependencies = [ + "bitflags 2.4.2", + "libc", +] + +[[package]] +name = "event-manager" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "90b16fe5161a1160c9c7cece9f7504f2412ef5e2c0643d1e322eccf37692a42b" +dependencies = [ + "libc", + "vmm-sys-util", +] + +[[package]] +name = "hashbrown" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" + +[[package]] +name = "hermit-abi" +version = "0.1.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "62b467343b94ba476dcb2500d242dadbb39557df889310ac77c5d99100aaac33" +dependencies = [ + "libc", +] + +[[package]] +name = "indexmap" +version = "1.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bd070e393353796e801d209ad339e89596eb4c8d430d18ede6a1cced8fafbd99" +dependencies = [ + "autocfg", + "hashbrown", +] + +[[package]] +name = "lazy_static" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" + +[[package]] +name = "libc" +version = "0.2.153" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c198f91728a82281a64e1f4f9eeb25d82cb32a5de251c6bd1b5154d63a8e7bd" + +[[package]] +name = "linked-hash-map" +version = "0.5.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0717cef1bc8b636c6e1c1bbdefc09e6322da8a9321966e8928ef80d20f7f770f" + +[[package]] +name = "log" +version = "0.4.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5e6163cb8c49088c2c36f57875e58ccd8c87c7427f7fbd50ea6710b2f3f2e8f" + +[[package]] +name = "os_str_bytes" +version = "6.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2355d85b9a3786f481747ced0e0ff2ba35213a1f9bd406ed906554d7af805a1" + +[[package]] +name = "proc-macro2" +version = "1.0.78" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2422ad645d89c99f8f3e6b88a9fdeca7fabeac836b1002371c4367c8f984aae" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "291ec9ab5efd934aaf503a6466c5d5251535d108ee747472c3977cc5acc868ef" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "ryu" +version = "1.0.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e86697c916019a8588c99b5fac3cead74ec0b4b819707a682fd4d23fa0ce1ba1" + +[[package]] +name = "seccompiler" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e01d1292a1131b22ccea49f30bd106f1238b5ddeec1a98d39268dcc31d540e68" +dependencies = [ + "libc", +] + +[[package]] +name = "serde" +version = "1.0.196" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "870026e60fa08c69f064aa766c10f10b1d62db9ccd4d0abb206472bee0ce3b32" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.196" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33c85360c95e7d137454dc81d9a4ed2b8efd8fbe19cee57357b32b9771fccb67" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_yaml" +version = "0.8.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "578a7433b776b56a35785ed5ce9a7e777ac0598aac5a6dd1b4b18a307c7fc71b" +dependencies = [ + "indexmap", + "ryu", + "serde", + "yaml-rust", +] + +[[package]] +name = "strsim" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623" + +[[package]] +name = "syn" +version = "2.0.49" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "915aea9e586f80826ee59f8453c1101f9d1c4b3964cd2460185ee8e299ada496" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "termcolor" +version = "1.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06794f8f6c5c898b3275aebefa6b8a1cb24cd2c6c79397ab15774837a0bc5755" +dependencies = [ + "winapi-util", +] + +[[package]] +name = "textwrap" +version = "0.16.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23d434d3f8967a09480fb04132ebe0a3e088c173e6d0ee7897abbdf4eab0f8b9" + +[[package]] +name = "thiserror" +version = "1.0.57" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e45bcbe8ed29775f228095caf2cd67af7a4ccf756ebff23a306bf3e8b47b24b" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "1.0.57" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a953cb265bef375dae3de6663da4d3804eee9682ea80d8e2542529b73c531c81" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "unicode-ident" +version = "1.0.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b" + +[[package]] +name = "vhost" +version = "0.10.0" +source = "git+https://github.com/joaopeixoto13/vhost?branch=vhost-user-frontend#cf0c49322d8bff5f55591421298c9cb6699279a1" +dependencies = [ + "bitflags 2.4.2", + "libc", + "vm-memory", + "vmm-sys-util", +] + +[[package]] +name = "vhost-user-frontend" +version = "0.1.0" +source = "git+https://github.com/joaopeixoto13/vhost?branch=vhost-user-frontend#cf0c49322d8bff5f55591421298c9cb6699279a1" +dependencies = [ + "epoll", + "libc", + "log", + "seccompiler", + "thiserror", + "vhost", + "virtio-bindings 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)", + "virtio-queue", + "vm-memory", + "vmm-sys-util", +] + +[[package]] +name = "virtio-bindings" +version = "0.2.2" + +[[package]] +name = "virtio-bindings" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "878bcb1b2812a10c30d53b0ed054999de3d98f25ece91fc173973f9c57aaae86" + +[[package]] +name = "virtio-blk" +version = "0.1.0" +dependencies = [ + "log", + "virtio-bindings 0.2.2", + "virtio-device", + "virtio-queue", + "vm-memory", + "vmm-sys-util", +] + +[[package]] +name = "virtio-device" +version = "0.1.0" +dependencies = [ + "log", + "virtio-bindings 0.2.2", + "virtio-queue", + "vm-memory", +] + +[[package]] +name = "virtio-queue" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3f69a13d6610db9312acbb438b0390362af905d37634a2106be70c0f734986d" +dependencies = [ + "log", + "virtio-bindings 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)", + "vm-memory", + "vmm-sys-util", +] + +[[package]] +name = "vm-device" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "599adbdaddea4947ca23c085d2b8e47f3499ccda35438424526f3853748a8eb6" + +[[package]] +name = "vm-memory" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "74ffc42216c32c35f858fa4bfdcd9b61017dfd691e0240268fdc85dbf59e5459" +dependencies = [ + "arc-swap", + "libc", + "thiserror", + "winapi", +] + +[[package]] +name = "vmm" +version = "0.1.0" +dependencies = [ + "api", + "devices", + "event-manager", + "libc", + "vm-device", + "vmm-sys-util", +] + +[[package]] +name = "vmm-sys-util" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d1435039746e20da4f8d507a72ee1b916f7b4b05af7a91c093d2c6561934ede" +dependencies = [ + "bitflags 1.3.2", + "libc", +] + +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] +name = "winapi-util" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f29e6f9198ba0d26b4c9f07dbe6f9ed633e1f3d5b8b414090084349e46a52596" +dependencies = [ + "winapi", +] + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + +[[package]] +name = "yaml-rust" +version = "0.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56c1936c4cc7a1c9ab21a1ebb602eb942ba868cbd44a99cb7cdc5892335e1c85" +dependencies = [ + "linked-hash-map", +] diff --git a/src/vmm/Cargo.toml b/src/vmm/Cargo.toml new file mode 100644 index 0000000..9d93621 --- /dev/null +++ b/src/vmm/Cargo.toml @@ -0,0 +1,18 @@ +[package] +name = "vmm" +version = "0.1.0" +edition = "2021" +authors = ["João Peixoto "] +keywords = ["bao", "VMM", "Virtualization"] +description = "Bao VMM" +license = "Apache-2.0" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +api = { path = "../api" } +virtio = { path = "../virtio" } +event-manager = { version = "0.4.0", features = ["remote_endpoint"] } +libc = ">=0.2.95" +vm-device = "0.1.0" +vmm-sys-util = "0.12.1" \ No newline at end of file diff --git a/src/vmm/src/lib.rs b/src/vmm/src/lib.rs new file mode 100644 index 0000000..095ff35 --- /dev/null +++ b/src/vmm/src/lib.rs @@ -0,0 +1,2 @@ +pub mod vm; +pub mod vmm; diff --git a/src/vmm/src/vm.rs b/src/vmm/src/vm.rs new file mode 100644 index 0000000..4afbdaa --- /dev/null +++ b/src/vmm/src/vm.rs @@ -0,0 +1,256 @@ +use api::defines::{BAO_IO_READ, BAO_IO_WRITE}; +use api::device_model::BaoDeviceModel; +use api::error::{Error, Result}; +use api::types::DeviceConfig; +use event_manager::{EventManager, MutEventSubscriber}; +use std::sync::{Arc, Mutex}; +use virtio::block::virtio::device::VirtioBlock; +use virtio::device::VirtioDeviceT; +use virtio::device::{VirtioDataPlane, VirtioDevType, VirtioDeviceType}; +use virtio::fs::vhost_user::device::VhostUserFs; +use virtio::net::vhost::device::VhostNet; +use virtio::net::virtio::device::VirtioNet; +use virtio::vsock::vhost::device::VhostVsockDevice; +use virtio::vsock::vhost_user::device::VhostUserVsock; +use vm_device::bus::MmioAddress; +use vm_device::device_manager::{IoManager, MmioManager}; + +/// Vm abstraction. +/// +/// # Attributes +/// +/// * `id` - The ID of the VM. +/// * `device_model` - The device model. +/// * `devices` - The list of devices. +/// * `device_manager` - The device manager responsible for providing methods for device registration, as well as for dispatching read and write requests. +/// * `event_manager` - The event manager responsible for handling and dispatch the device events. +pub struct Vm { + pub id: u16, + device_model: Arc>, + devices: Vec, + device_manager: Arc>, + event_manager: Arc>>>>, +} + +impl Vm { + /// Create a new VM. + /// + /// # Arguments + /// + /// * `fd` - The file descriptor for the VMM. + /// * `config` - The device configuration. + /// + /// # Returns + /// + /// A `Result` containing the result of the operation. + pub fn new(fd: i32, config: DeviceConfig) -> Result { + // Create the device manager. + let device_manager = Arc::new(Mutex::new(IoManager::new())); + + // Create the event manager. + let event_manager = Arc::new(Mutex::new( + EventManager::>>::new() + .map_err(Error::EventManager)?, + )); + + // Create the VM. + let mut vm = Vm { + id: config.id as u16, + device_model: Arc::new(Mutex::new( + BaoDeviceModel::new(fd, config.id as u16).unwrap(), + )), + devices: Vec::new(), + device_manager, + event_manager, + }; + + // Add the device. + // FIXME: For now one VM can have only one device. + vm.add_device(&config).unwrap(); + + Ok(vm) + } + + /// Add a new device. + /// + /// # Arguments + /// + /// * `config` - The device configuration. + /// + /// # Returns + /// + /// A `Result` containing the result of the operation. + fn add_device(&mut self, config: &DeviceConfig) -> Result<()> { + // Extract the device type. + let device_type = VirtioDevType::from(config.device_type.as_str()); + + // Extract the data plane. + let data_plane = VirtioDataPlane::from(config.data_plane.as_str()); + + // Clone the device manager, event manager and device model. + let device_manager = self.device_manager.clone(); + let event_manager = self.event_manager.clone(); + let device_model = self.device_model.clone(); + + let device = match device_type { + // Block device. + VirtioDevType::Block => match data_plane { + VirtioDataPlane::Virtio => Ok(VirtioDeviceType::VirtioBlock( + VirtioBlock::new(config, device_manager, event_manager, device_model).unwrap(), + )), + _ => Err(Error::WrongDeviceConfiguration( + VirtioDevType::to_string(&device_type), + VirtioDataPlane::to_string(&data_plane), + )), + }, + // Virtual Filesystem device. + VirtioDevType::Fs => match data_plane { + VirtioDataPlane::VhostUser => Ok(VirtioDeviceType::VhostUserFs( + VhostUserFs::new(config, device_manager, event_manager, device_model).unwrap(), + )), + _ => Err(Error::WrongDeviceConfiguration( + VirtioDevType::to_string(&device_type), + VirtioDataPlane::to_string(&data_plane), + )), + }, + // Vsock device. + VirtioDevType::Vsock => match data_plane { + VirtioDataPlane::Vhost => Ok(VirtioDeviceType::VhostVsock( + VhostVsockDevice::new(config, device_manager, event_manager, device_model) + .unwrap(), + )), + VirtioDataPlane::VhostUser => Ok(VirtioDeviceType::VhostUserVsock( + VhostUserVsock::new(config, device_manager, event_manager, device_model) + .unwrap(), + )), + _ => Err(Error::WrongDeviceConfiguration( + VirtioDevType::to_string(&device_type), + VirtioDataPlane::to_string(&data_plane), + )), + }, + // Network device. + VirtioDevType::Net => match data_plane { + VirtioDataPlane::Virtio => Ok(VirtioDeviceType::VirtioNet( + VirtioNet::new(config, device_manager, event_manager, device_model).unwrap(), + )), + VirtioDataPlane::Vhost => Ok(VirtioDeviceType::VhostNet( + VhostNet::new(config, device_manager, event_manager, device_model).unwrap(), + )), + _ => Err(Error::WrongDeviceConfiguration( + VirtioDevType::to_string(&device_type), + VirtioDataPlane::to_string(&data_plane), + )), + }, + // Other device types. + _ => Err(Error::WrongDeviceConfiguration( + VirtioDevType::to_string(&device_type), + VirtioDataPlane::to_string(&data_plane), + )), + } + .unwrap(); + + // Push the device to the list of devices. + self.devices.push(device); + + Ok(()) + } + + /// Run the I/O events. + /// + /// # Returns + /// + /// A `Result` containing the result of the operation. + pub fn run_io(self: Arc) -> Result<()> { + loop { + //Attach the I/O client. + match self.device_model.lock().unwrap().attach_io_client() { + Ok(()) => {} + Err(err) => { + return Err(err); + } + } + + // Request the I/O client + let mut req = match self.device_model.lock().unwrap().request_io() { + Ok(req) => req, + Err(err) => { + return Err(err); + } + }; + + //Call the device manager to dispatch the I/O request + let mut data = (req.value as u32).to_le_bytes(); + + match req.op { + BAO_IO_WRITE => { + match self + .device_manager + .lock() + .unwrap() + .mmio_write(MmioAddress(req.addr), &mut data) + { + Ok(()) => {} + Err(_err) => { + println!("Invalid Mmio write operation"); + return Err(Error::InvalidMmioOperation("write")); + } + } + } + BAO_IO_READ => { + match self + .device_manager + .lock() + .unwrap() + .mmio_read(MmioAddress(req.addr), &mut data) + { + Ok(()) => {} + Err(_err) => { + println!("Invalid Mmio read operation"); + return Err(Error::InvalidMmioOperation("read")); + } + } + } + _ => { + println!("Invalid I/O request direction: {:?}", req.op); + return Err(Error::InvalidIoReqDirection(req.op)); + } + } + + // Update the req.value with the data. + req.value = u32::from_le_bytes(data) as u64; + + // Notify the I/O client that the I/O request has been completed + match self.device_model.lock().unwrap().notify_io_completed(req) { + Ok(()) => {} + Err(err) => { + return Err(err); + } + } + } + } + + /// Run the event manager. + /// + /// # Note + /// + /// This method is responsible for running the event manager loop + /// on a different thread to handle the device events (Ioevenfds) + /// and to dispatch the respective I/O events to the associated + /// device. + pub fn run_event_manager(self: Arc) { + loop { + self.event_manager.lock().unwrap().run().unwrap(); + } + } +} + +// Implementing `Send` trait unsafely for `Vm`. +// This indicates it's considered safe to transfer `Vm` instances between threads, +// enabling transferring ownership of a `Vm` instance between threads. +// As the `Vm` instance is protected by a Mutex, it's safe to transfer ownership of it between threads. +unsafe impl Send for Vm {} + +// Implementing `Sync` trait unsafely for `Vm`. +// This indicates it's considered safe to share references of `Vm` between threads. +// As the `Vm` instance is protected by a Mutex, it's safe to share references of it between threads. +unsafe impl Sync for Vm {} diff --git a/src/vmm/src/vmm.rs b/src/vmm/src/vmm.rs new file mode 100644 index 0000000..bd968a6 --- /dev/null +++ b/src/vmm/src/vmm.rs @@ -0,0 +1,105 @@ +use api::error::{Error, Result}; +use api::types::VMMConfig; +use std::fs::OpenOptions; +use std::os::fd::AsRawFd; +use std::sync::{Arc, Mutex}; +use std::thread::{Builder, JoinHandle}; + +use super::vm::Vm; + +/// VMM abstraction. +/// +/// # Attributes +/// +/// * `fd` - The file descriptor for the VMM (e.g. /dev/bao). +/// * `vms` - The list of VMs. +/// * `vcpus` - The list of vCPUs/threads. +pub struct Vmm { + fd: i32, + vms: Mutex>>, + vcpus: Mutex>>, +} + +impl TryFrom for Vmm { + type Error = Error; + + /// Try_from method used to create a VMM from a VMM configuration. + /// + /// # Arguments + /// + /// * `config` - The VMM configuration. + /// + /// # Returns + /// + /// A `Result` containing the result of the operation. + fn try_from(config: VMMConfig) -> Result { + // Open the VMM file descriptor. + let fd = OpenOptions::new() + .read(true) + .write(true) + .open("/dev/bao") + .map_err(|_e| Error::OpenFdFailed("/dev/bao", std::io::Error::last_os_error()))?; + + // Create the VMM. + let vmm = Vmm { + fd: fd.as_raw_fd(), + vms: Mutex::new(Vec::new()), + vcpus: Mutex::new(Vec::new()), + }; + + // Create all VMs. + for config in config.devices { + let vm = Vm::new(vmm.fd, config).unwrap(); + + // Add the VM to the VMM list. + vmm.vms.lock().unwrap().push(Arc::new(vm)); + } + + Ok(vmm) + } +} + +impl Vmm { + /// Run the VMM. + /// + /// # Returns + /// + /// A `Result` containing the result of the operation. + pub fn run(&self) -> Result<()> { + for vm in self.vms.lock().unwrap().drain(..) { + // Create a new vCPU/thread to run the I/O events. + let vm_io = vm.clone(); + self.vcpus.lock().unwrap().push( + Builder::new() + .name(format!("vm_{}_io", vm_io.id)) + .spawn(move || { + vm_io.run_io().unwrap(); + }) + .unwrap(), + ); + + // Create a new vCPU/thread to run the VM event manager. + let vm_evm = vm.clone(); + self.vcpus.lock().unwrap().push( + Builder::new() + .name(format!("vm_{}_evm", vm_evm.id)) + .spawn(move || { + vm_evm.run_event_manager(); + }) + .unwrap(), + ); + } + Ok(()) + } +} + +impl Drop for Vmm { + /// Drops all handles from the vcpus vector. + fn drop(&mut self) { + // Loops until all handles are popped from the vcpus vector + while let Some(handle) = self.vcpus.lock().unwrap().pop() { + // Joins the thread represented by the handle + handle.join().unwrap(); + } + } +}