From 2ba9389761efaa56864e1e9d703b37a304444679 Mon Sep 17 00:00:00 2001 From: David Irvine Date: Mon, 18 Nov 2024 19:50:16 +0000 Subject: [PATCH] fix: update benchmarks to use public API This commit updates the benchmarks to use the public API instead of internal functions: - Replace `decrypt_full_set` with public `decrypt` function in benchmarks - Update imports in benches/lib.rs to use only public API - Maintain same benchmarking functionality while using proper public interface The change ensures that benchmarks follow the same patterns as regular usage of the library by utilizing the public API instead of internal implementation details. This makes the benchmarks more representative of real-world usage and maintains proper API boundaries. --- benches/lib.rs | 4 +- examples/basic_encryptor.rs | 5 +- examples/parallel_streaming_decryptor.rs | 22 ++--- src/decrypt.rs | 6 +- src/lib.rs | 27 +++--- src/python.rs | 49 ++++++----- tests/integration_tests.rs | 101 +++++++++++++++++------ 7 files changed, 127 insertions(+), 87 deletions(-) diff --git a/benches/lib.rs b/benches/lib.rs index e3f67a5af..3a2b171c1 100644 --- a/benches/lib.rs +++ b/benches/lib.rs @@ -44,7 +44,7 @@ #![allow(missing_copy_implementations, missing_debug_implementations)] use criterion::{BatchSize, Bencher, Criterion}; -use self_encryption::{decrypt_full_set, encrypt, test_helpers::random_bytes}; +use self_encryption::{decrypt, encrypt, test_helpers::random_bytes}; use std::time::Duration; // sample size is _NOT_ the number of times the command is run... @@ -75,7 +75,7 @@ fn read(b: &mut Bencher, bytes_len: usize) { || encrypt(random_bytes(bytes_len)).unwrap(), // actual benchmark |(data_map, encrypted_chunks)| { - let _raw_data = decrypt_full_set(&data_map, &encrypted_chunks).unwrap(); + let _raw_data = decrypt(&data_map, &encrypted_chunks).unwrap(); }, BatchSize::SmallInput, ); diff --git a/examples/basic_encryptor.rs b/examples/basic_encryptor.rs index f412ba4df..e1b65bec7 100644 --- a/examples/basic_encryptor.rs +++ b/examples/basic_encryptor.rs @@ -41,9 +41,9 @@ use bytes::Bytes; use docopt::Docopt; use rayon::prelude::*; +use self_encryption::decrypt; use self_encryption::{ - self, DataMap, EncryptedChunk, Error, Result, - encrypt, serialize, deserialize + self, deserialize, encrypt, serialize, DataMap, EncryptedChunk, Error, Result, }; use serde::Deserialize; use std::{ @@ -56,7 +56,6 @@ use std::{ sync::Arc, }; use xor_name::XorName; -use self_encryption::decrypt; #[rustfmt::skip] static USAGE: &str = " diff --git a/examples/parallel_streaming_decryptor.rs b/examples/parallel_streaming_decryptor.rs index 8b6d15a1d..24751a36e 100644 --- a/examples/parallel_streaming_decryptor.rs +++ b/examples/parallel_streaming_decryptor.rs @@ -1,13 +1,7 @@ use bytes::Bytes; -use self_encryption::{ - streaming_decrypt_from_storage, DataMap, Error, Result, deserialize -}; -use std::{ - fs::File, - io::Read, - path::Path, -}; use rayon::prelude::*; +use self_encryption::{deserialize, streaming_decrypt_from_storage, DataMap, Error, Result}; +use std::{fs::File, io::Read, path::Path}; use xor_name::XorName; fn main() -> Result<()> { @@ -31,21 +25,17 @@ fn main() -> Result<()> { }; // Use the streaming decryption function - streaming_decrypt_from_storage( - &data_map, - Path::new("output_file.dat"), - get_chunk_parallel, - )?; + streaming_decrypt_from_storage(&data_map, Path::new("output_file.dat"), get_chunk_parallel)?; Ok(()) } // Helper function to load data map from a file fn load_data_map(path: &str) -> Result { - let mut file = File::open(path) - .map_err(|e| Error::Generic(format!("Failed to open data map: {}", e)))?; + let mut file = + File::open(path).map_err(|e| Error::Generic(format!("Failed to open data map: {}", e)))?; let mut data = Vec::new(); file.read_to_end(&mut data) .map_err(|e| Error::Generic(format!("Failed to read data map: {}", e)))?; deserialize(&data) -} \ No newline at end of file +} diff --git a/src/decrypt.rs b/src/decrypt.rs index be18bbb2a..4f76d88e8 100644 --- a/src/decrypt.rs +++ b/src/decrypt.rs @@ -29,11 +29,7 @@ pub fn decrypt_sorted_set( /// Decrypt a chunk, given the index of that chunk in the sequence of chunks, /// and the raw encrypted content. -pub fn decrypt_chunk( - chunk_index: usize, - content: &Bytes, - src_hashes: &[XorName], -) -> Result { +pub fn decrypt_chunk(chunk_index: usize, content: &Bytes, src_hashes: &[XorName]) -> Result { let pki = get_pad_key_and_iv(chunk_index, src_hashes); let (pad, key, iv) = pki; diff --git a/src/lib.rs b/src/lib.rs index 8f846d5e8..add4f433f 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -105,7 +105,6 @@ pub use decrypt::decrypt_chunk; use utils::*; pub use xor_name::XorName; - pub use self::{ data_map::{ChunkInfo, DataMap}, error::{Error, Result}, @@ -218,10 +217,7 @@ pub fn encrypt(bytes: Bytes) -> Result<(DataMap, Vec)> { /// # Returns /// /// * `Result` - The decrypted data or an error if chunks are missing/corrupted -pub(crate) fn decrypt_full_set( - data_map: &DataMap, - chunks: &[EncryptedChunk], -) -> Result { +pub(crate) fn decrypt_full_set(data_map: &DataMap, chunks: &[EncryptedChunk]) -> Result { let src_hashes = extract_hashes(data_map); // Create a mapping of chunk hashes to chunks for efficient lookup @@ -505,7 +501,9 @@ where for (info, chunk) in root_map.infos().iter().zip(encrypted_chunks.iter()) { let decrypted_chunk = decrypt_chunk(info.index, &chunk.content, &src_hashes)?; - output_file.write_all(&decrypted_chunk).map_err(Error::from)?; + output_file + .write_all(&decrypted_chunk) + .map_err(Error::from)?; } Ok(()) @@ -524,10 +522,7 @@ where /// # Returns /// /// * `Result` - The root data map or an error if retrieval or decryption fails. -pub fn get_root_data_map_parallel( - data_map: DataMap, - get_chunk_parallel: &F, -) -> Result +pub fn get_root_data_map_parallel(data_map: DataMap, get_chunk_parallel: &F) -> Result where F: Fn(&[XorName]) -> Result>, { @@ -598,8 +593,7 @@ where /// /// * `Result>` - The serialized bytes or an error pub fn serialize(data: &T) -> Result> { - bincode::serialize(data) - .map_err(|e| Error::Generic(format!("Serialization error: {}", e))) + bincode::serialize(data).map_err(|e| Error::Generic(format!("Serialization error: {}", e))) } /// Deserializes bytes into a data structure using bincode. @@ -612,8 +606,7 @@ pub fn serialize(data: &T) -> Result> { /// /// * `Result` - The deserialized data structure or an error pub fn deserialize(bytes: &[u8]) -> Result { - bincode::deserialize(bytes) - .map_err(|e| Error::Generic(format!("Deserialization error: {}", e))) + bincode::deserialize(bytes).map_err(|e| Error::Generic(format!("Deserialization error: {}", e))) } /// Verifies and deserializes a chunk by checking its content hash matches the provided name. @@ -632,10 +625,10 @@ pub fn verify_chunk(name: XorName, bytes: &[u8]) -> Result { let chunk = EncryptedChunk { content: Bytes::from(bytes.to_vec()), }; - + // Calculate the hash of the encrypted content directly let calculated_hash = XorName::from_content(chunk.content.as_ref()); - + // Verify the hash matches if calculated_hash != name { return Err(Error::Generic(format!( @@ -643,7 +636,7 @@ pub fn verify_chunk(name: XorName, bytes: &[u8]) -> Result { name, calculated_hash ))); } - + Ok(chunk) } diff --git a/src/python.rs b/src/python.rs index f55b0b03e..ee01b4d31 100644 --- a/src/python.rs +++ b/src/python.rs @@ -1,11 +1,9 @@ use crate::{ - decrypt as rust_decrypt, - decrypt_from_storage as rust_decrypt_from_storage, - encrypt as rust_encrypt, - encrypt_from_file as rust_encrypt_from_file, + decrypt as rust_decrypt, decrypt_from_storage as rust_decrypt_from_storage, + encrypt as rust_encrypt, encrypt_from_file as rust_encrypt_from_file, shrink_data_map as rust_shrink_data_map, - streaming_decrypt_from_storage as rust_streaming_decrypt_from_storage, - ChunkInfo, DataMap as RustDataMap, EncryptedChunk as RustEncryptedChunk, Error, Result, + streaming_decrypt_from_storage as rust_streaming_decrypt_from_storage, ChunkInfo, + DataMap as RustDataMap, EncryptedChunk as RustEncryptedChunk, Error, Result, }; use bytes::Bytes; use pyo3::prelude::*; @@ -140,24 +138,28 @@ fn encrypt(_py: Python<'_>, data: &PyBytes) -> PyResult<(PyDataMap, Vec(e.to_string()))?; - + Ok(( PyDataMap { inner: data_map }, - chunks.into_iter().map(|c| PyEncryptedChunk { inner: c }).collect(), + chunks + .into_iter() + .map(|c| PyEncryptedChunk { inner: c }) + .collect(), )) } #[pyfunction] fn encrypt_from_file(input_path: String, output_dir: String) -> PyResult<(PyDataMap, Vec)> { - let (data_map, chunk_names) = rust_encrypt_from_file( - &PathBuf::from(input_path), - &PathBuf::from(output_dir), - ) - .map_err(|e| PyErr::new::(e.to_string()))?; + let (data_map, chunk_names) = + rust_encrypt_from_file(&PathBuf::from(input_path), &PathBuf::from(output_dir)) + .map_err(|e| PyErr::new::(e.to_string()))?; Ok(( PyDataMap { inner: data_map }, - chunk_names.into_iter().map(|name| hex::encode(name.0)).collect(), + chunk_names + .into_iter() + .map(|name| hex::encode(name.0)) + .collect(), )) } @@ -166,7 +168,7 @@ fn decrypt(data_map: &PyDataMap, chunks: Vec) -> PyResult = chunks.into_iter().map(|c| c.inner).collect(); let result = rust_decrypt(&data_map.inner, &chunks) .map_err(|e| PyErr::new::(e.to_string()))?; - + Python::with_gil(|py| Ok(PyBytes::new(py, &result).into())) } @@ -212,7 +214,10 @@ fn shrink_data_map( Ok(( PyDataMap { inner: shrunk_map }, - chunks.into_iter().map(|c| PyEncryptedChunk { inner: c }).collect(), + chunks + .into_iter() + .map(|c| PyEncryptedChunk { inner: c }) + .collect(), )) } @@ -234,15 +239,21 @@ fn streaming_decrypt_from_storage( Ok(chunk_data.into_iter().map(Bytes::from).collect()) }; - rust_streaming_decrypt_from_storage(&data_map.inner, &PathBuf::from(output_path), get_chunk_parallel) - .map_err(|e| PyErr::new::(e.to_string())) + rust_streaming_decrypt_from_storage( + &data_map.inner, + &PathBuf::from(output_path), + get_chunk_parallel, + ) + .map_err(|e| PyErr::new::(e.to_string())) } #[pyfunction] fn verify_chunk(name: &PyXorName, content: &PyBytes) -> PyResult { match crate::verify_chunk(name.inner, content.as_bytes()) { Ok(chunk) => Ok(PyEncryptedChunk { inner: chunk }), - Err(e) => Err(PyErr::new::(e.to_string())), + Err(e) => Err(PyErr::new::( + e.to_string(), + )), } } diff --git a/tests/integration_tests.rs b/tests/integration_tests.rs index 50af30c3b..1c75c1c05 100644 --- a/tests/integration_tests.rs +++ b/tests/integration_tests.rs @@ -1,7 +1,9 @@ use bytes::Bytes; +use rayon::prelude::*; use self_encryption::{ decrypt, decrypt_from_storage, encrypt, encrypt_from_file, get_root_data_map, shrink_data_map, - streaming_decrypt_from_storage, test_helpers::random_bytes, verify_chunk, DataMap, EncryptedChunk, Error, Result, + streaming_decrypt_from_storage, test_helpers::random_bytes, verify_chunk, DataMap, + EncryptedChunk, Error, Result, }; use std::{ collections::HashMap, @@ -11,7 +13,6 @@ use std::{ }; use tempfile::TempDir; use xor_name::XorName; -use rayon::prelude::*; // Define traits for our storage operations type StoreFn = Box Result<()>>; @@ -464,10 +465,10 @@ fn test_comprehensive_encryption_decryption() -> Result<()> { // Test sizes to ensure we test both small and large files let test_cases = vec![ - ("3MB", 3 * 1024 * 1024), // Basic 3-chunk case - ("5MB", 5 * 1024 * 1024), // Triggers shrinking - ("10MB", 10 * 1024 * 1024), // Larger file - ("20MB", 20 * 1024 * 1024), // Even larger file + ("3MB", 3 * 1024 * 1024), // Basic 3-chunk case + ("5MB", 5 * 1024 * 1024), // Triggers shrinking + ("10MB", 10 * 1024 * 1024), // Larger file + ("20MB", 20 * 1024 * 1024), // Even larger file ]; for (size_name, size) in test_cases { @@ -494,7 +495,11 @@ fn test_comprehensive_encryption_decryption() -> Result<()> { // A. Test decrypt() with in-memory encryption result println!("\nA.1 Testing decrypt() with encrypt() result:"); let decrypted_a1 = decrypt(&data_map1, &chunks1)?; - assert_eq!(original_data.as_ref(), decrypted_a1.as_ref(), "Mismatch: encrypt() -> decrypt()"); + assert_eq!( + original_data.as_ref(), + decrypted_a1.as_ref(), + "Mismatch: encrypt() -> decrypt()" + ); println!("✓ decrypt() successful"); // B. Test decrypt_from_storage() with in-memory encryption result @@ -511,17 +516,24 @@ fn test_comprehensive_encryption_decryption() -> Result<()> { let mut decrypted = Vec::new(); File::open(&output_path1)?.read_to_end(&mut decrypted)?; - assert_eq!(original_data.as_ref(), decrypted.as_slice(), "Mismatch: encrypt() -> decrypt_from_storage()"); + assert_eq!( + original_data.as_ref(), + decrypted.as_slice(), + "Mismatch: encrypt() -> decrypt_from_storage()" + ); println!("✓ decrypt_from_storage() successful"); // C. Test streaming_decrypt_from_storage() with in-memory encryption result println!("\nA.3 Testing streaming_decrypt_from_storage() with encrypt() result:"); - let output_path1_stream = temp_dir.path().join(format!("output1_stream_{}.dat", size_name)); - + let output_path1_stream = temp_dir + .path() + .join(format!("output1_stream_{}.dat", size_name)); + // Create parallel chunk retrieval function let chunk_dir = storage.disk_dir.path().to_owned(); let get_chunk_parallel = |hashes: &[XorName]| -> Result> { - hashes.par_iter() + hashes + .par_iter() .map(|hash| { let chunk_path = chunk_dir.join(hex::encode(hash)); let mut chunk_data = Vec::new(); @@ -537,7 +549,11 @@ fn test_comprehensive_encryption_decryption() -> Result<()> { let mut decrypted = Vec::new(); File::open(&output_path1_stream)?.read_to_end(&mut decrypted)?; - assert_eq!(original_data.as_ref(), decrypted.as_slice(), "Mismatch: encrypt() -> streaming_decrypt_from_storage()"); + assert_eq!( + original_data.as_ref(), + decrypted.as_slice(), + "Mismatch: encrypt() -> streaming_decrypt_from_storage()" + ); println!("✓ streaming_decrypt_from_storage() successful"); // D. Test decrypt() with file-based encryption result @@ -552,7 +568,11 @@ fn test_comprehensive_encryption_decryption() -> Result<()> { }); } let decrypted2 = decrypt(&data_map2, &file_chunks)?; - assert_eq!(original_data.as_ref(), decrypted2.as_ref(), "Mismatch: encrypt_from_file() -> decrypt()"); + assert_eq!( + original_data.as_ref(), + decrypted2.as_ref(), + "Mismatch: encrypt_from_file() -> decrypt()" + ); println!("✓ decrypt() successful"); // E. Test decrypt_from_storage() with file-based encryption result @@ -563,35 +583,59 @@ fn test_comprehensive_encryption_decryption() -> Result<()> { let mut decrypted = Vec::new(); File::open(&output_path2)?.read_to_end(&mut decrypted)?; - assert_eq!(original_data.as_ref(), decrypted.as_slice(), "Mismatch: encrypt_from_file() -> decrypt_from_storage()"); + assert_eq!( + original_data.as_ref(), + decrypted.as_slice(), + "Mismatch: encrypt_from_file() -> decrypt_from_storage()" + ); println!("✓ decrypt_from_storage() successful"); // F. Test streaming_decrypt_from_storage() with file-based encryption result println!("\nB.3 Testing streaming_decrypt_from_storage() with encrypt_from_file() result:"); - let output_path2_stream = temp_dir.path().join(format!("output2_stream_{}.dat", size_name)); + let output_path2_stream = temp_dir + .path() + .join(format!("output2_stream_{}.dat", size_name)); streaming_decrypt_from_storage(&data_map2, &output_path2_stream, &get_chunk_parallel)?; let mut decrypted = Vec::new(); File::open(&output_path2_stream)?.read_to_end(&mut decrypted)?; - assert_eq!(original_data.as_ref(), decrypted.as_slice(), "Mismatch: encrypt_from_file() -> streaming_decrypt_from_storage()"); + assert_eq!( + original_data.as_ref(), + decrypted.as_slice(), + "Mismatch: encrypt_from_file() -> streaming_decrypt_from_storage()" + ); println!("✓ streaming_decrypt_from_storage() successful"); // Additional verifications println!("\n=== Verifying consistency ==="); - + // Verify data maps are equivalent - assert_eq!(data_map1.len(), data_map2.len(), "Data maps have different number of chunks"); - assert_eq!(data_map1.child(), data_map2.child(), "Data maps have different child levels"); + assert_eq!( + data_map1.len(), + data_map2.len(), + "Data maps have different number of chunks" + ); + assert_eq!( + data_map1.child(), + data_map2.child(), + "Data maps have different child levels" + ); println!("✓ Data maps match"); // Verify chunk counts - assert_eq!(chunks1.len(), file_chunks.len(), "Different number of chunks between methods"); + assert_eq!( + chunks1.len(), + file_chunks.len(), + "Different number of chunks between methods" + ); println!("✓ Chunk counts match"); // Verify all output files are identical let outputs = vec![ - output_path1, output_path1_stream, - output_path2, output_path2_stream + output_path1, + output_path1_stream, + output_path2, + output_path2_stream, ]; for (i, path1) in outputs.iter().enumerate() { for path2 in outputs.iter().skip(i + 1) { @@ -599,7 +643,11 @@ fn test_comprehensive_encryption_decryption() -> Result<()> { let mut content2 = Vec::new(); File::open(path1)?.read_to_end(&mut content1)?; File::open(path2)?.read_to_end(&mut content2)?; - assert_eq!(content1, content2, "Output files don't match: {:?} vs {:?}", path1, path2); + assert_eq!( + content1, content2, + "Output files don't match: {:?} vs {:?}", + path1, path2 + ); } } println!("✓ All output files match"); @@ -668,7 +716,10 @@ fn test_chunk_verification() -> Result<()> { // Get the first chunk info and content let first_chunk_info = &data_map.infos()[0]; - let chunk_path = storage.disk_dir.path().join(hex::encode(first_chunk_info.dst_hash)); + let chunk_path = storage + .disk_dir + .path() + .join(hex::encode(first_chunk_info.dst_hash)); let mut chunk_content = Vec::new(); File::open(&chunk_path)?.read_to_end(&mut chunk_content)?; @@ -704,7 +755,7 @@ fn test_chunk_verification() -> Result<()> { let chunk_path = storage.disk_dir.path().join(hex::encode(info.dst_hash)); let mut chunk_content = Vec::new(); File::open(&chunk_path)?.read_to_end(&mut chunk_content)?; - + match verify_chunk(info.dst_hash, &chunk_content) { Ok(_) => println!("✓ Chunk {} verified successfully", i), Err(e) => println!("✗ Chunk {} verification failed: {}", i, e),