Feat/stone ffi

Cargo.toml

@@ -2,6 +2,7 @@
 members = [
     "boulder",
     "moss",
+    "libstone",
     "crates/*",
 ]
 default-members = [
@@ -57,6 +58,8 @@ xxhash-rust = { version = "0.8.11", features = ["xxh3"] }
 zstd = { version = "0.13.2", features = ["zstdmt"] }
 mailparse = "0.15.0"
 zbus = "4.4.0"
+libc = "0.2.62"
+cbindgen = "0.26"
 
 [profile.release]
 lto = "thin"

boulder/src/package.rs

@@ -6,9 +6,9 @@ use std::{io, num::NonZeroU64};
 
 use fs_err as fs;
 use itertools::Itertools;
+use stone::StoneDigestWriterHasher;
 use thiserror::Error;
 
-use stone::write::digest;
 use stone_recipe::{script, Package};
 
 use crate::{build, container, timing, util, Macros, Paths, Recipe, Timing};
@@ -60,7 +60,7 @@ impl<'a> Packager<'a> {
 
     pub fn package(&self, timing: &mut Timing) -> Result<(), Error> {
         // Hasher used for calculating file digests
-        let mut hasher = digest::Hasher::new();
+        let mut hasher = StoneDigestWriterHasher::new();
 
         let timer = timing.begin(timing::Kind::Analyze);
 

boulder/src/package/analysis.rs

@@ -9,7 +9,7 @@ use std::{
 };
 
 use moss::{Dependency, Provider};
-use stone::write::digest;
+use stone::StoneDigestWriterHasher;
 use tui::{ProgressBar, ProgressStyle, Styled};
 
 use crate::{Paths, Recipe};
@@ -25,12 +25,17 @@ pub struct Chain<'a> {
     recipe: &'a Recipe,
     paths: &'a Paths,
     collector: &'a Collector,
-    hasher: &'a mut digest::Hasher,
+    hasher: &'a mut StoneDigestWriterHasher,
     pub buckets: BTreeMap<String, Bucket>,
 }
 
 impl<'a> Chain<'a> {
-    pub fn new(paths: &'a Paths, recipe: &'a Recipe, collector: &'a Collector, hasher: &'a mut digest::Hasher) -> Self {
+    pub fn new(
+        paths: &'a Paths,
+        recipe: &'a Recipe,
+        collector: &'a Collector,
+        hasher: &'a mut StoneDigestWriterHasher,
+    ) -> Self {
         Self {
             handlers: vec![
                 Box::new(handler::ignore_blocked),
@@ -144,7 +149,7 @@ impl Bucket {
 pub struct BucketMut<'a> {
     pub providers: &'a mut BTreeSet<Provider>,
     pub dependencies: &'a mut BTreeSet<Dependency>,
-    pub hasher: &'a mut digest::Hasher,
+    pub hasher: &'a mut StoneDigestWriterHasher,
     pub recipe: &'a Recipe,
     pub paths: &'a Paths,
 }

boulder/src/package/collect.rs

@@ -12,8 +12,7 @@ use std::{
 use fs_err as fs;
 use glob::Pattern;
 use nix::libc::{S_IFDIR, S_IRGRP, S_IROTH, S_IRWXU, S_IXGRP, S_IXOTH};
-use stone::payload::{layout, Layout};
-use stone::write::digest;
+use stone::{StoneDigestWriter, StoneDigestWriterHasher, StonePayloadLayoutRecord, StonePayloadLayoutFile};
 use thiserror::Error;
 
 #[derive(Debug, Clone, Eq, PartialEq)]
@@ -61,7 +60,7 @@ impl Collector {
     }
 
     /// Produce a [`PathInfo`] from the provided [`Path`]
-    pub fn path(&self, path: &Path, hasher: &mut digest::Hasher) -> Result<PathInfo, Error> {
+    pub fn path(&self, path: &Path, hasher: &mut StoneDigestWriterHasher) -> Result<PathInfo, Error> {
         let metadata = fs::metadata(path)?;
         self.path_with_metadata(path.to_path_buf(), &metadata, hasher)
     }
@@ -70,7 +69,7 @@ impl Collector {
         &self,
         path: PathBuf,
         metadata: &Metadata,
-        hasher: &mut digest::Hasher,
+        hasher: &mut StoneDigestWriterHasher,
     ) -> Result<PathInfo, Error> {
         let target_path = Path::new("/").join(path.strip_prefix(&self.root).expect("path is ancestor of root"));
 
@@ -85,7 +84,7 @@ impl Collector {
     pub fn enumerate_paths(
         &self,
         subdir: Option<(PathBuf, Metadata)>,
-        hasher: &mut digest::Hasher,
+        hasher: &mut StoneDigestWriterHasher,
     ) -> Result<Vec<PathInfo>, Error> {
         let mut paths = vec![];
 
@@ -127,7 +126,7 @@ impl Collector {
 pub struct PathInfo {
     pub path: PathBuf,
     pub target_path: PathBuf,
-    pub layout: Layout,
+    pub layout: StonePayloadLayoutRecord,
     pub size: u64,
     pub package: String,
 }
@@ -137,7 +136,7 @@ impl PathInfo {
         path: PathBuf,
         target_path: PathBuf,
         metadata: &Metadata,
-        hasher: &mut digest::Hasher,
+        hasher: &mut StoneDigestWriterHasher,
         package: String,
     ) -> Result<Self, Error> {
         let layout = layout_from_metadata(&path, &target_path, metadata, hasher)?;
@@ -151,15 +150,15 @@ impl PathInfo {
         })
     }
 
-    pub fn restat(&mut self, hasher: &mut digest::Hasher) -> Result<(), Error> {
+    pub fn restat(&mut self, hasher: &mut StoneDigestWriterHasher) -> Result<(), Error> {
         let metadata = fs::metadata(&self.path)?;
         self.layout = layout_from_metadata(&self.path, &self.target_path, &metadata, hasher)?;
         self.size = metadata.size();
         Ok(())
     }
 
     pub fn is_file(&self) -> bool {
-        matches!(self.layout.entry, layout::Entry::Regular(_, _))
+        matches!(self.layout.file, StonePayloadLayoutFile::Regular(_, _))
     }
 
     pub fn file_name(&self) -> &str {
@@ -180,8 +179,8 @@ fn layout_from_metadata(
     path: &Path,
     target_path: &Path,
     metadata: &Metadata,
-    hasher: &mut digest::Hasher,
-) -> Result<Layout, Error> {
+    hasher: &mut StoneDigestWriterHasher,
+) -> Result<StonePayloadLayoutRecord, Error> {
     // Strip /usr
     let target = target_path
         .strip_prefix("/usr")
@@ -191,29 +190,29 @@ fn layout_from_metadata(
 
     let file_type = metadata.file_type();
 
-    Ok(Layout {
+    Ok(StonePayloadLayoutRecord {
         uid: metadata.uid(),
         gid: metadata.gid(),
         mode: metadata.mode(),
         tag: 0,
-        entry: if file_type.is_symlink() {
+        file: if file_type.is_symlink() {
             let source = fs::read_link(path)?;
 
-            layout::Entry::Symlink(source.to_string_lossy().to_string(), target)
+            StonePayloadLayoutFile::Symlink(source.to_string_lossy().to_string(), target)
         } else if file_type.is_dir() {
-            layout::Entry::Directory(target)
+            StonePayloadLayoutFile::Directory(target)
         } else if file_type.is_char_device() {
-            layout::Entry::CharacterDevice(target)
+            StonePayloadLayoutFile::CharacterDevice(target)
         } else if file_type.is_block_device() {
-            layout::Entry::BlockDevice(target)
+            StonePayloadLayoutFile::BlockDevice(target)
         } else if file_type.is_fifo() {
-            layout::Entry::Fifo(target)
+            StonePayloadLayoutFile::Fifo(target)
         } else if file_type.is_socket() {
-            layout::Entry::Socket(target)
+            StonePayloadLayoutFile::Socket(target)
         } else {
             hasher.reset();
 
-            let mut digest_writer = digest::Writer::new(io::sink(), hasher);
+            let mut digest_writer = StoneDigestWriter::new(io::sink(), hasher);
             let mut file = fs::File::open(path)?;
 
             // Copy bytes to null sink so we don't
@@ -222,7 +221,7 @@ fn layout_from_metadata(
 
             let hash = hasher.digest128();
 
-            layout::Entry::Regular(hash, target)
+            StonePayloadLayoutFile::Regular(hash, target)
         },
     })
 }

boulder/src/package/emit.rs

@@ -10,6 +10,7 @@ use std::{
 use fs_err::{self as fs, File};
 use itertools::Itertools;
 use moss::{package::Meta, Dependency, Provider};
+use stone::{StoneHeaderV1FileType, StoneWriteError, StoneWriter};
 use thiserror::Error;
 use tui::{ProgressBar, ProgressStyle, Styled};
 
@@ -166,7 +167,7 @@ fn emit_package(paths: &Paths, package: &Package) -> Result<(), Error> {
     let mut out_file = File::create(out_path)?;
 
     // Create stone binary writer
-    let mut writer = stone::Writer::new(&mut out_file, stone::header::v1::FileType::Binary)?;
+    let mut writer = StoneWriter::new(&mut out_file, StoneHeaderV1FileType::Binary)?;
 
     // Add metadata
     {
@@ -227,7 +228,7 @@ fn emit_package(paths: &Paths, package: &Package) -> Result<(), Error> {
 #[derive(Debug, Error)]
 pub enum Error {
     #[error("stone binary writer")]
-    StoneBinaryWriter(#[from] stone::write::Error),
+    StoneBinaryWriter(#[from] StoneWriteError),
     #[error("manifest")]
     Manifest(#[from] manifest::Error),
     #[error("io")]

boulder/src/package/emit/manifest.rs

@@ -4,6 +4,7 @@
 
 use std::{collections::BTreeSet, io, path::PathBuf};
 
+use stone::StoneWriteError;
 use thiserror::Error;
 
 use crate::{Architecture, Paths, Recipe};
@@ -69,7 +70,7 @@ impl<'a> Manifest<'a> {
 #[derive(Debug, Error)]
 pub enum Error {
     #[error("stone binary writer")]
-    StoneWriter(#[from] stone::write::Error),
+    StoneWriter(#[from] StoneWriteError),
     #[error("encode json")]
     Json(#[from] serde_json::Error),
     #[error("io")]

boulder/src/package/emit/manifest/binary.rs

@@ -7,8 +7,7 @@ use std::{collections::BTreeSet, path::Path};
 use fs_err::File;
 use moss::Dependency;
 use stone::{
-    header::v1::FileType,
-    payload::{self, meta},
+    StoneHeaderV1FileType, StonePayloadMetaPrimitive, StonePayloadMetaRecord, StonePayloadMetaTag, StoneWriter,
 };
 
 use super::Error;
@@ -17,7 +16,7 @@ use crate::package::emit::Package;
 pub fn write(path: &Path, packages: &BTreeSet<&Package>, build_deps: &BTreeSet<String>) -> Result<(), Error> {
     let mut output = File::create(path)?;
 
-    let mut writer = stone::Writer::new(&mut output, FileType::BuildManifest)?;
+    let mut writer = StoneWriter::new(&mut output, StoneHeaderV1FileType::BuildManifest)?;
 
     // Add each package
     for package in packages {
@@ -29,9 +28,9 @@ pub fn write(path: &Path, packages: &BTreeSet<&Package>, build_deps: &BTreeSet<S
         // Add build deps
         for name in build_deps {
             if let Ok(dep) = Dependency::from_name(name) {
-                payload.push(payload::Meta {
-                    tag: meta::Tag::BuildDepends,
-                    kind: meta::Kind::Dependency(dep.kind.into(), dep.name),
+                payload.push(StonePayloadMetaRecord {
+                    tag: StonePayloadMetaTag::BuildDepends,
+                    primitive: StonePayloadMetaPrimitive::Dependency(dep.kind.into(), dep.name),
                 });
             }
         }

boulder/src/package/emit/manifest/json.rs

@@ -42,7 +42,7 @@ pub fn write(
                 .analysis
                 .paths
                 .iter()
-                .map(|p| format!("/usr/{}", p.layout.entry.target()))
+                .map(|p| format!("/usr/{}", p.layout.file.target()))
                 .sorted()
                 .collect();
 

crates/stone/Cargo.toml

@@ -6,6 +6,10 @@ rust-version.workspace = true
 
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
+[features]
+default = []
+ffi = []
+
 [dependencies]
 strum.workspace = true
 thiserror.workspace = true

crates/stone/benches/read.rs

@@ -9,6 +9,7 @@ use std::{
 };
 
 use criterion::{black_box, criterion_group, criterion_main, Criterion};
+use stone::StoneDecodedPayload;
 
 fn read_unbuffered(path: impl AsRef<Path>) {
     read(File::open(path).unwrap());
@@ -23,7 +24,7 @@ fn read<R: Read + Seek>(reader: R) {
 
     let payloads = stone.payloads().unwrap().collect::<Result<Vec<_>, _>>().unwrap();
 
-    if let Some(content) = payloads.iter().find_map(stone::read::PayloadKind::content) {
+    if let Some(content) = payloads.iter().find_map(StoneDecodedPayload::content) {
         stone.unpack_content(content, &mut sink()).unwrap();
     }
 }

crates/stone/src/ext.rs

@@ -0,0 +1,77 @@
+use std::io::{Read, Result, Write};
+
+pub trait ReadExt: Read {
+    fn read_u8(&mut self) -> Result<u8> {
+        let bytes = self.read_array::<1>()?;
+        Ok(bytes[0])
+    }
+
+    fn read_u16(&mut self) -> Result<u16> {
+        let bytes = self.read_array()?;
+        Ok(u16::from_be_bytes(bytes))
+    }
+
+    fn read_u32(&mut self) -> Result<u32> {
+        let bytes = self.read_array()?;
+        Ok(u32::from_be_bytes(bytes))
+    }
+
+    fn read_u64(&mut self) -> Result<u64> {
+        let bytes = self.read_array()?;
+        Ok(u64::from_be_bytes(bytes))
+    }
+
+    fn read_u128(&mut self) -> Result<u128> {
+        let bytes = self.read_array()?;
+        Ok(u128::from_be_bytes(bytes))
+    }
+
+    fn read_array<const N: usize>(&mut self) -> Result<[u8; N]> {
+        let mut bytes = [0u8; N];
+        self.read_exact(&mut bytes)?;
+        Ok(bytes)
+    }
+
+    fn read_vec(&mut self, length: usize) -> Result<Vec<u8>> {
+        let mut bytes = vec![0u8; length];
+        self.read_exact(&mut bytes)?;
+        Ok(bytes)
+    }
+
+    fn read_string(&mut self, length: u64) -> Result<String> {
+        let mut string = String::with_capacity(length as usize);
+        self.take(length).read_to_string(&mut string)?;
+        Ok(string)
+    }
+}
+
+impl<T: Read> ReadExt for T {}
+
+pub trait WriteExt: Write {
+    fn write_u8(&mut self, item: u8) -> Result<()> {
+        self.write_array([item])
+    }
+
+    fn write_u16(&mut self, item: u16) -> Result<()> {
+        self.write_array(item.to_be_bytes())
+    }
+
+    fn write_u32(&mut self, item: u32) -> Result<()> {
+        self.write_array(item.to_be_bytes())
+    }
+
+    fn write_u64(&mut self, item: u64) -> Result<()> {
+        self.write_array(item.to_be_bytes())
+    }
+
+    fn write_u128(&mut self, item: u128) -> Result<()> {
+        self.write_array(item.to_be_bytes())
+    }
+
+    fn write_array<const N: usize>(&mut self, bytes: [u8; N]) -> Result<()> {
+        self.write_all(&bytes)?;
+        Ok(())
+    }
+}
+
+impl<T: Write> WriteExt for T {}