From 48e80209edee9a85b8bbfb0d203f531cfb14ec15 Mon Sep 17 00:00:00 2001 From: Timo Tiuraniemi Date: Wed, 20 Jul 2022 12:52:02 +0300 Subject: [PATCH 001/157] fix: tests compile only by temporarily allowing dead code for page_len --- src/bitfield/mod.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/bitfield/mod.rs b/src/bitfield/mod.rs index b261c2a6..9546a1e2 100644 --- a/src/bitfield/mod.rs +++ b/src/bitfield/mod.rs @@ -30,6 +30,7 @@ use std::ops::Range; pub struct Bitfield { data: SparseBitfield, index: SparseBitfield, + #[allow(dead_code)] page_len: u64, length: u64, masks: Masks, From e154f793548974a4829a607d6781be805f716a8b Mon Sep 17 00:00:00 2001 From: Timo Tiuraniemi Date: Mon, 1 Aug 2022 16:20:39 +0300 Subject: [PATCH 002/157] test: add skeleton for Javascript interop tests --- .gitignore | 2 ++ Cargo.toml | 7 +++++++ tests/common/mod.rs | 31 ++++++++++++++++++++++++++++++ tests/js/interop.js | 36 +++++++++++++++++++++++++++++++++++ tests/js/mod.rs | 44 +++++++++++++++++++++++++++++++++++++++++++ tests/js/package.json | 10 ++++++++++ tests/js_interop.rs | 26 +++++++++++++++++++++++++ 7 files changed, 156 insertions(+) create mode 100644 tests/js/interop.js create mode 100644 tests/js/mod.rs create mode 100644 tests/js/package.json create mode 100644 tests/js_interop.rs diff --git a/.gitignore b/.gitignore index 2e7f0592..292f6504 100644 --- a/.gitignore +++ b/.gitignore @@ -7,7 +7,9 @@ npm-debug.log* .nyc_output target/ Cargo.lock +package-lock.json my-first-dataset/ feed.db/ .vscode +tests/js/work diff --git a/Cargo.toml b/Cargo.toml index e36d9d89..0f18f936 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -45,3 +45,10 @@ data-encoding = "2.2.0" remove_dir_all = "0.7.0" tempfile = "3.1.0" async-std = { version = "1.5.0", features = ["attributes"] } +sha2 = "0.10.2" + +[features] +# Used only in interoperability tests under tests/js-interop which use the javascript version of hypercore +# to verify that this crate works. To run them, use: +# cargo test --features js-interop-tests +js_interop_tests = [] diff --git a/tests/common/mod.rs b/tests/common/mod.rs index fa43ddb0..64aa44c1 100644 --- a/tests/common/mod.rs +++ b/tests/common/mod.rs @@ -4,9 +4,40 @@ use anyhow::Error; use futures::future::FutureExt; use hypercore::{Feed, Storage, Store}; use random_access_memory as ram; +use sha2::{Digest, Sha256}; pub async fn create_feed(page_size: usize) -> Result, Error> { let create = |_store: Store| async move { Ok(ram::RandomAccessMemory::new(page_size)) }.boxed(); let storage = Storage::new(create, false).await?; Feed::with_storage(storage).await } + +#[derive(PartialEq, Debug)] +pub struct HypercoreHash { + pub bitfield: String, + pub data: String, + pub oplog: String, + pub tree: String, +} + +pub fn create_hypercore_hash(dir: String) -> Result { + let bitfield = hash_file(format!("{}/bitfield", dir))?; + let data = hash_file(format!("{}/data", dir))?; + let oplog = hash_file(format!("{}/oplog", dir))?; + let tree = hash_file(format!("{}/tree", dir))?; + Ok(HypercoreHash { + bitfield, + data, + oplog, + tree, + }) +} + +pub fn hash_file(file: String) -> Result { + let mut hasher = Sha256::new(); + let mut file = std::fs::File::open(file)?; + + std::io::copy(&mut file, &mut hasher)?; + let hash_bytes = hasher.finalize(); + Ok(format!("{:X}", hash_bytes)) +} diff --git a/tests/js/interop.js b/tests/js/interop.js new file mode 100644 index 00000000..f60fe148 --- /dev/null +++ b/tests/js/interop.js @@ -0,0 +1,36 @@ +const Hypercore = require('hypercore'); + +// Static test key pair obtained with: +// +// const crypto = require('hypercore-crypto'); +// const keyPair = crypto.keyPair(); +// console.log("public key", keyPair.publicKey.toString('hex').match(/../g).join(' ')); +// console.log("secret key", keyPair.secretKey.toString('hex').match(/../g).join(' ')); +const testKeyPair = { + publicKey: Buffer.from([ + 0x97, 0x60, 0x6c, 0xaa, 0xd2, 0xb0, 0x8c, 0x1d, 0x5f, 0xe1, 0x64, 0x2e, 0xee, 0xa5, 0x62, 0xcb, + 0x91, 0xd6, 0x55, 0xe2, 0x00, 0xc8, 0xd4, 0x3a, 0x32, 0x09, 0x1d, 0x06, 0x4a, 0x33, 0x1e, 0xe3]), + secretKey: Buffer.from([ + 0x27, 0xe6, 0x74, 0x25, 0xc1, 0xff, 0xd1, 0xd9, 0xee, 0x62, 0x5c, 0x96, 0x2b, 0x57, 0x13, 0xc3, + 0x51, 0x0b, 0x71, 0x14, 0x15, 0xf3, 0x31, 0xf6, 0xfa, 0x9e, 0xf2, 0xbf, 0x23, 0x5f, 0x2f, 0xfe, + 0x97, 0x60, 0x6c, 0xaa, 0xd2, 0xb0, 0x8c, 0x1d, 0x5f, 0xe1, 0x64, 0x2e, 0xee, 0xa5, 0x62, 0xcb, + 0x91, 0xd6, 0x55, 0xe2, 0x00, 0xc8, 0xd4, 0x3a, 0x32, 0x09, 0x1d, 0x06, 0x4a, 0x33, 0x1e, 0xe3]), +} + +if (process.argv.length !== 4) { + console.error("Usage: node interop.js [test step] [test set]") + process.exit(1); +} + +if (process.argv[2] === '1') { + step1(process.argv[3]); +} else { + console.error(`Invalid test step {}`, process.argv[2]); + process.exit(2); +} + +async function step1(testSet) { + const core = new Hypercore(`work/${testSet}`, testKeyPair.publicKey, {keyPair: testKeyPair}); + await core.append(['Hello', 'World']) + await core.close() +}; diff --git a/tests/js/mod.rs b/tests/js/mod.rs new file mode 100644 index 00000000..8356e72b --- /dev/null +++ b/tests/js/mod.rs @@ -0,0 +1,44 @@ +use std::fs::{create_dir_all, remove_dir_all, remove_file}; +use std::path::Path; +use std::process::Command; + +pub fn cleanup() { + if Path::new("tests/js/node_modules").exists() { + remove_dir_all("tests/js/node_modules").expect("Unable to run rm to delete node_modules"); + } + + if Path::new("tests/js/work").exists() { + remove_dir_all("tests/js/work").expect("Unable to run rm to delete work"); + } + if Path::new("tests/js/package-lock.json").exists() { + remove_file("tests/js/package-lock.json") + .expect("Unable to run rm to delete package-lock.json"); + } +} + +pub fn init(test_set: &str) { + let status = Command::new("npm") + .current_dir("tests/js") + .args(&["install"]) + .status() + .expect("Unable to run npm install"); + assert_eq!( + Some(0), + status.code(), + "npm install did not run successfully. Do you have npm installed and a network connection?" + ); + create_dir_all(format!("tests/js/work/{}", test_set)).expect("Unable to create work directory"); +} + +pub fn step_1_create_hypercore(test_set: &str) { + let status = Command::new("npm") + .current_dir("tests/js") + .args(&["run", "step1", test_set]) + .status() + .expect("Unable to run npm run"); + assert_eq!( + Some(0), + status.code(), + "node step 1 did not run successfully" + ); +} diff --git a/tests/js/package.json b/tests/js/package.json new file mode 100644 index 00000000..8dc6d71b --- /dev/null +++ b/tests/js/package.json @@ -0,0 +1,10 @@ +{ + "name": "hypercore-js-interop-tests", + "version": "0.0.1", + "scripts": { + "step1": "node interop.js 1" + }, + "dependencies": { + "hypercore": "next" + } +} diff --git a/tests/js_interop.rs b/tests/js_interop.rs new file mode 100644 index 00000000..fc71639c --- /dev/null +++ b/tests/js_interop.rs @@ -0,0 +1,26 @@ +mod common; +mod js; +use js::{cleanup, init, step_1_create_hypercore}; + +const WORK_DIR: &str = "tests/js/work"; +const TEST_SET_BASIC: &str = "basic"; + +#[test] +#[cfg_attr(not(feature = "js_interop_tests"), ignore)] +fn basic_interop_with_javascript() { + cleanup(); + init(TEST_SET_BASIC); + step_1_create_hypercore(TEST_SET_BASIC); + let hash = common::create_hypercore_hash(format!("{}/{}", WORK_DIR, TEST_SET_BASIC)) + .expect("Could not hash directory"); + assert_eq!(get_step_1_hash(), hash) +} + +fn get_step_1_hash() -> common::HypercoreHash { + common::HypercoreHash { + bitfield: "0E2E1FF956A39192CBB68D2212288FE75B32733AB0C442B9F0471E254A0382A2".into(), + data: "872E4E50CE9990D8B041330C47C9DDD11BEC6B503AE9386A99DA8584E9BB12C4".into(), + oplog: "E374F3CFEA62D333E3ADE22A24A0EA50E5AF09CF45E2DEDC0F56F5A214081156".into(), + tree: "8577B24ADC763F65D562CD11204F938229AD47F27915B0821C46A0470B80813A".into(), + } +} From 7d7a8095ff9065aed98546e106aba7c216ed3204 Mon Sep 17 00:00:00 2001 From: Timo Tiuraniemi Date: Fri, 5 Aug 2022 13:21:54 +0300 Subject: [PATCH 003/157] Add beginnings of test for rust creation of the same feed --- src/storage/mod.rs | 15 ++++++++++ tests/compat.rs | 1 + tests/js/interop.js | 6 +++- tests/js/mod.rs | 11 ++++++-- tests/js_interop.rs | 68 +++++++++++++++++++++++++++++++++++++++------ 5 files changed, 88 insertions(+), 13 deletions(-) diff --git a/src/storage/mod.rs b/src/storage/mod.rs index 0d5fa2ba..4ee6752a 100644 --- a/src/storage/mod.rs +++ b/src/storage/mod.rs @@ -42,6 +42,8 @@ pub enum Store { Signatures, /// Keypair Keypair, + /// Oplog + Oplog, } /// Save data to a desired storage backend. @@ -55,6 +57,7 @@ where bitfield: T, signatures: T, keypair: T, + oplog: T, } impl Storage @@ -74,14 +77,25 @@ where let bitfield = create(Store::Bitfield).await?; let signatures = create(Store::Signatures).await?; let keypair = create(Store::Keypair).await?; + let oplog = create(Store::Oplog).await?; let mut instance = Self { tree, data, bitfield, signatures, keypair, + oplog, }; + if overwrite || instance.bitfield.len().await.unwrap_or(0) == 0 { + // TODO: This is + instance + .oplog + .write(0, &[0x00]) + .await + .map_err(|e| anyhow!(e))?; + } + if overwrite || instance.bitfield.len().await.unwrap_or(0) == 0 { let header = create_bitfield(); instance @@ -394,6 +408,7 @@ impl Storage { Store::Bitfield => "bitfield", Store::Signatures => "signatures", Store::Keypair => "key", + Store::Oplog => "oplog", }; RandomAccessDisk::open(dir.as_path().join(name)).boxed() }; diff --git a/tests/compat.rs b/tests/compat.rs index 93b6ec34..e68c4f6d 100644 --- a/tests/compat.rs +++ b/tests/compat.rs @@ -142,6 +142,7 @@ fn storage_path>(dir: P, s: Store) -> PathBuf { Store::Bitfield => "bitfield", Store::Signatures => "signatures", Store::Keypair => "key", + Store::Oplog => "oplog", }; dir.as_ref().join(filename) } diff --git a/tests/js/interop.js b/tests/js/interop.js index f60fe148..6bce6d0d 100644 --- a/tests/js/interop.js +++ b/tests/js/interop.js @@ -17,6 +17,10 @@ const testKeyPair = { 0x91, 0xd6, 0x55, 0xe2, 0x00, 0xc8, 0xd4, 0x3a, 0x32, 0x09, 0x1d, 0x06, 0x4a, 0x33, 0x1e, 0xe3]), } + +const crypto = require('hypercore-crypto'); +console.log("HELLO SIGN", crypto.sign(new TextEncoder().encode('hello'), testKeyPair.secretKey).toString('hex').match(/../g).join(' ')); + if (process.argv.length !== 4) { console.error("Usage: node interop.js [test step] [test set]") process.exit(1); @@ -31,6 +35,6 @@ if (process.argv[2] === '1') { async function step1(testSet) { const core = new Hypercore(`work/${testSet}`, testKeyPair.publicKey, {keyPair: testKeyPair}); - await core.append(['Hello', 'World']) + await core.append(['Hello', 'World']); await core.close() }; diff --git a/tests/js/mod.rs b/tests/js/mod.rs index 8356e72b..8b312a19 100644 --- a/tests/js/mod.rs +++ b/tests/js/mod.rs @@ -16,7 +16,7 @@ pub fn cleanup() { } } -pub fn init(test_set: &str) { +pub fn install() { let status = Command::new("npm") .current_dir("tests/js") .args(&["install"]) @@ -27,10 +27,15 @@ pub fn init(test_set: &str) { status.code(), "npm install did not run successfully. Do you have npm installed and a network connection?" ); - create_dir_all(format!("tests/js/work/{}", test_set)).expect("Unable to create work directory"); } -pub fn step_1_create_hypercore(test_set: &str) { +pub fn prepare_test_set(test_set: &str) -> String { + let path = format!("tests/js/work/{}", test_set); + create_dir_all(&path).expect("Unable to create work directory"); + path +} + +pub fn js_step_1_create_hypercore(test_set: &str) { let status = Command::new("npm") .current_dir("tests/js") .args(&["run", "step1", test_set]) diff --git a/tests/js_interop.rs b/tests/js_interop.rs index fc71639c..c746c0cc 100644 --- a/tests/js_interop.rs +++ b/tests/js_interop.rs @@ -1,21 +1,71 @@ mod common; mod js; -use js::{cleanup, init, step_1_create_hypercore}; +use std::{path::Path, sync::Once}; -const WORK_DIR: &str = "tests/js/work"; -const TEST_SET_BASIC: &str = "basic"; +use ed25519_dalek::{PublicKey, SecretKey, PUBLIC_KEY_LENGTH, SECRET_KEY_LENGTH}; +use hypercore::{FeedBuilder, Storage}; +use js::{cleanup, install, js_step_1_create_hypercore, prepare_test_set}; + +const TEST_SET_JS_FIRST: &str = "jsfirst"; +const TEST_SET_RS_FIRST: &str = "rsfirst"; + +const TEST_PUBLIC_KEY_BYTES: [u8; PUBLIC_KEY_LENGTH] = [ + 0x97, 0x60, 0x6c, 0xaa, 0xd2, 0xb0, 0x8c, 0x1d, 0x5f, 0xe1, 0x64, 0x2e, 0xee, 0xa5, 0x62, 0xcb, + 0x91, 0xd6, 0x55, 0xe2, 0x00, 0xc8, 0xd4, 0x3a, 0x32, 0x09, 0x1d, 0x06, 0x4a, 0x33, 0x1e, 0xe3, +]; +// NB: In the javascript version this is 64 bytes, but that's because sodium appends the the public +// key after the secret key for some reason. Only the first 32 bytes are actually used in +// javascript side too for signing. +const TEST_SECRET_KEY_BYTES: [u8; SECRET_KEY_LENGTH] = [ + 0x27, 0xe6, 0x74, 0x25, 0xc1, 0xff, 0xd1, 0xd9, 0xee, 0x62, 0x5c, 0x96, 0x2b, 0x57, 0x13, 0xc3, + 0x51, 0x0b, 0x71, 0x14, 0x15, 0xf3, 0x31, 0xf6, 0xfa, 0x9e, 0xf2, 0xbf, 0x23, 0x5f, 0x2f, 0xfe, +]; + +static INIT: Once = Once::new(); +fn init() { + INIT.call_once(|| { + // run initialization here + cleanup(); + install(); + }); +} #[test] #[cfg_attr(not(feature = "js_interop_tests"), ignore)] -fn basic_interop_with_javascript() { - cleanup(); - init(TEST_SET_BASIC); - step_1_create_hypercore(TEST_SET_BASIC); - let hash = common::create_hypercore_hash(format!("{}/{}", WORK_DIR, TEST_SET_BASIC)) - .expect("Could not hash directory"); +fn js_interop_js_first() { + init(); + let work_dir = prepare_test_set(TEST_SET_JS_FIRST); + js_step_1_create_hypercore(TEST_SET_JS_FIRST); + let hash = common::create_hypercore_hash(work_dir).expect("Could not hash directory"); assert_eq!(get_step_1_hash(), hash) } +#[async_std::test] +#[cfg_attr(not(feature = "js_interop_tests"), ignore)] +async fn js_interop_rs_first() { + init(); + let work_dir = prepare_test_set(TEST_SET_RS_FIRST); + step_1_create_hypercore(&work_dir).await; + let hash = common::create_hypercore_hash(work_dir).expect("Could not hash directory"); + // TODO: Make this match, only data does right now + // assert_eq!(get_step_1_hash(), hash) +} + +async fn step_1_create_hypercore(work_dir: &str) { + let path = Path::new(work_dir).to_owned(); + let storage = Storage::new_disk(&path, false).await.unwrap(); + + let public_key = PublicKey::from_bytes(&TEST_PUBLIC_KEY_BYTES).unwrap(); + let secret_key = SecretKey::from_bytes(&TEST_SECRET_KEY_BYTES).unwrap(); + + let builder = FeedBuilder::new(public_key, storage); + let mut feed = builder.secret_key(secret_key).build().await.unwrap(); + + feed.append(b"Hello").await.unwrap(); + feed.append(b"World").await.unwrap(); + drop(feed); +} + fn get_step_1_hash() -> common::HypercoreHash { common::HypercoreHash { bitfield: "0E2E1FF956A39192CBB68D2212288FE75B32733AB0C442B9F0471E254A0382A2".into(), From ae4f7f365c79f49c51adc741d8fd7de759ba68f7 Mon Sep 17 00:00:00 2001 From: Timo Tiuraniemi Date: Tue, 16 Aug 2022 16:54:42 +0300 Subject: [PATCH 004/157] Beginnings of compact_encoding primitives --- src/compact_encoding/mod.rs | 58 +++++++++++++++++++++++++++++++++++++ src/lib.rs | 1 + tests/compact_encoding.rs | 14 +++++++++ tests/js/interop.js | 24 ++++++++++----- tests/js/package.json | 2 +- tests/js_interop.rs | 2 +- 6 files changed, 91 insertions(+), 10 deletions(-) create mode 100644 src/compact_encoding/mod.rs create mode 100644 tests/compact_encoding.rs diff --git a/src/compact_encoding/mod.rs b/src/compact_encoding/mod.rs new file mode 100644 index 00000000..24d01771 --- /dev/null +++ b/src/compact_encoding/mod.rs @@ -0,0 +1,58 @@ +//! Compact encoding module. Rust implementation of https://github.com/compact-encoding/compact-encoding. + +use std::fmt::Debug; + +/// State. +#[derive(Debug)] +pub struct CencState { + /// Start position + pub start: usize, + /// End position + pub end: usize, + /// Buffer to hold the encoding + pub buffer: Vec, +} + +impl CencState { + /// Create emtpy state + pub fn new() -> CencState { + CencState { + start: 0, + end: 0, + buffer: vec![], + } + } +} + +/// Compact Encoding +pub trait CompactEncoding +where + T: Debug, +{ + /// Preencode + fn preencode(state: CencState, value: T) -> CencState; + + // /// Encode + // fn encode(state: CencState, value: T); + + // /// Decode + // fn decode(state: CencState) -> T; +} + +/// Compact Encoder +#[derive(Debug)] +pub struct CompactEncoder {} + +impl CompactEncoding for CompactEncoder { + fn preencode(mut state: CencState, value: String) -> CencState { + state.end += value.len(); + state + } +} + +impl CompactEncoding for CompactEncoder { + fn preencode(mut state: CencState, _value: u64) -> CencState { + state.end += 8; + state + } +} diff --git a/src/lib.rs b/src/lib.rs index 0c99939c..3a822861 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -31,6 +31,7 @@ //! [Feed]: crate::feed::Feed pub mod bitfield; +pub mod compact_encoding; pub mod prelude; mod audit; diff --git a/tests/compact_encoding.rs b/tests/compact_encoding.rs new file mode 100644 index 00000000..185f05d3 --- /dev/null +++ b/tests/compact_encoding.rs @@ -0,0 +1,14 @@ +use hypercore::compact_encoding::CompactEncoding; +use hypercore::compact_encoding::{CencState, CompactEncoder}; + +#[test] +fn cenc_create() { + let state = CencState::new(); + assert_eq!(state.start, 0); + assert_eq!(state.end, 0); + assert_eq!(state.buffer.capacity(), 0); + let state = CompactEncoder::preencode(state, "test".to_string()); + assert_eq!(state.start, 0); + assert_eq!(state.end, 4); + assert_eq!(state.buffer.capacity(), 0); +} diff --git a/tests/js/interop.js b/tests/js/interop.js index 6bce6d0d..dfc7cc82 100644 --- a/tests/js/interop.js +++ b/tests/js/interop.js @@ -17,24 +17,32 @@ const testKeyPair = { 0x91, 0xd6, 0x55, 0xe2, 0x00, 0xc8, 0xd4, 0x3a, 0x32, 0x09, 0x1d, 0x06, 0x4a, 0x33, 0x1e, 0xe3]), } - -const crypto = require('hypercore-crypto'); -console.log("HELLO SIGN", crypto.sign(new TextEncoder().encode('hello'), testKeyPair.secretKey).toString('hex').match(/../g).join(' ')); - if (process.argv.length !== 4) { console.error("Usage: node interop.js [test step] [test set]") process.exit(1); } if (process.argv[2] === '1') { - step1(process.argv[3]); + step1(process.argv[3]).then(result => { + console.log("step1 ready", result); + }); } else { console.error(`Invalid test step {}`, process.argv[2]); process.exit(2); } async function step1(testSet) { - const core = new Hypercore(`work/${testSet}`, testKeyPair.publicKey, {keyPair: testKeyPair}); - await core.append(['Hello', 'World']); - await core.close() + let core = new Hypercore(`work/${testSet}`, testKeyPair.publicKey, {keyPair: testKeyPair}); + let len = await core.append(['Hello', 'World', 'foo', 'bar']); + console.log("LEN", len); + len = await core.append(['zan', 'tup']); + console.log("LEN2", len); + len = await core.append(['fup']); + console.log("LEN3", len); + len = await core.append(['sup']); + console.log("LEN4", len); + // len = await core.append(['flushes']); + // console.log("LEN5", len); + await core.close(); + core = new Hypercore(`work/${testSet}`, testKeyPair.publicKey, {keyPair: testKeyPair}); }; diff --git a/tests/js/package.json b/tests/js/package.json index 8dc6d71b..1088532b 100644 --- a/tests/js/package.json +++ b/tests/js/package.json @@ -5,6 +5,6 @@ "step1": "node interop.js 1" }, "dependencies": { - "hypercore": "next" + "hypercore": "file:../../../hypercore-js" } } diff --git a/tests/js_interop.rs b/tests/js_interop.rs index c746c0cc..6add0da1 100644 --- a/tests/js_interop.rs +++ b/tests/js_interop.rs @@ -37,7 +37,7 @@ fn js_interop_js_first() { let work_dir = prepare_test_set(TEST_SET_JS_FIRST); js_step_1_create_hypercore(TEST_SET_JS_FIRST); let hash = common::create_hypercore_hash(work_dir).expect("Could not hash directory"); - assert_eq!(get_step_1_hash(), hash) + // assert_eq!(get_step_1_hash(), hash) } #[async_std::test] From 307626865d333985880f38d28c1225dd62742c0a Mon Sep 17 00:00:00 2001 From: Timo Tiuraniemi Date: Wed, 17 Aug 2022 08:25:33 +0300 Subject: [PATCH 005/157] Nicer API --- src/compact_encoding/mod.rs | 32 +++++++++++++------------------- tests/compact_encoding.rs | 7 +++---- 2 files changed, 16 insertions(+), 23 deletions(-) diff --git a/src/compact_encoding/mod.rs b/src/compact_encoding/mod.rs index 24d01771..71117806 100644 --- a/src/compact_encoding/mod.rs +++ b/src/compact_encoding/mod.rs @@ -4,7 +4,7 @@ use std::fmt::Debug; /// State. #[derive(Debug)] -pub struct CencState { +pub struct State { /// Start position pub start: usize, /// End position @@ -13,10 +13,10 @@ pub struct CencState { pub buffer: Vec, } -impl CencState { +impl State { /// Create emtpy state - pub fn new() -> CencState { - CencState { + pub fn new() -> State { + State { start: 0, end: 0, buffer: vec![], @@ -30,29 +30,23 @@ where T: Debug, { /// Preencode - fn preencode(state: CencState, value: T) -> CencState; + fn preencode(&mut self, value: T); // /// Encode - // fn encode(state: CencState, value: T); + // fn encode(state: State, value: T); // /// Decode - // fn decode(state: CencState) -> T; + // fn decode(state: State) -> T; } -/// Compact Encoder -#[derive(Debug)] -pub struct CompactEncoder {} - -impl CompactEncoding for CompactEncoder { - fn preencode(mut state: CencState, value: String) -> CencState { - state.end += value.len(); - state +impl CompactEncoding for State { + fn preencode(&mut self, value: String) { + self.end += value.len(); } } -impl CompactEncoding for CompactEncoder { - fn preencode(mut state: CencState, _value: u64) -> CencState { - state.end += 8; - state +impl CompactEncoding for State { + fn preencode(&mut self, _value: u64) { + self.end += 8; } } diff --git a/tests/compact_encoding.rs b/tests/compact_encoding.rs index 185f05d3..926868a3 100644 --- a/tests/compact_encoding.rs +++ b/tests/compact_encoding.rs @@ -1,13 +1,12 @@ -use hypercore::compact_encoding::CompactEncoding; -use hypercore::compact_encoding::{CencState, CompactEncoder}; +use hypercore::compact_encoding::{CompactEncoding, State}; #[test] fn cenc_create() { - let state = CencState::new(); + let mut state = State::new(); assert_eq!(state.start, 0); assert_eq!(state.end, 0); assert_eq!(state.buffer.capacity(), 0); - let state = CompactEncoder::preencode(state, "test".to_string()); + state.preencode("test".to_string()); assert_eq!(state.start, 0); assert_eq!(state.end, 4); assert_eq!(state.buffer.capacity(), 0); From 0756a1d34ea3de13cf8c3aaa15807946a3fd6eb3 Mon Sep 17 00:00:00 2001 From: Timo Tiuraniemi Date: Wed, 17 Aug 2022 11:07:51 +0300 Subject: [PATCH 006/157] Efficient API where the buffer is owned by the client --- src/compact_encoding/mod.rs | 44 +++++++++++++++++++++++-------------- tests/compact_encoding.rs | 23 ++++++++++++++----- 2 files changed, 46 insertions(+), 21 deletions(-) diff --git a/src/compact_encoding/mod.rs b/src/compact_encoding/mod.rs index 71117806..c7ead70b 100644 --- a/src/compact_encoding/mod.rs +++ b/src/compact_encoding/mod.rs @@ -9,18 +9,30 @@ pub struct State { pub start: usize, /// End position pub end: usize, - /// Buffer to hold the encoding - pub buffer: Vec, } impl State { /// Create emtpy state pub fn new() -> State { - State { - start: 0, - end: 0, - buffer: vec![], - } + State { start: 0, end: 0 } + } + + /// Create a state with an already known size. + /// With this, you can/must skip the preencode step. + pub fn new_with_size(size: usize) -> (State, Box<[u8]>) { + ( + State { + start: 0, + end: size, + }, + vec![0; size].into_boxed_slice(), + ) + } + + /// After calling preencode(), this allocates the right size buffer to the heap. + /// Follow this with the same number of encode() steps to fill the created buffer. + pub fn create_buffer(&self) -> Box<[u8]> { + vec![0; self.end].into_boxed_slice() } } @@ -30,23 +42,23 @@ where T: Debug, { /// Preencode - fn preencode(&mut self, value: T); + fn preencode(&mut self, value: &T); - // /// Encode - // fn encode(state: State, value: T); + /// Encode + fn encode(&mut self, value: &T, buffer: &mut Box<[u8]>); // /// Decode // fn decode(state: State) -> T; } -impl CompactEncoding for State { - fn preencode(&mut self, value: String) { +impl CompactEncoding<&str> for State { + fn preencode(&mut self, value: &&str) { self.end += value.len(); } -} -impl CompactEncoding for State { - fn preencode(&mut self, _value: u64) { - self.end += 8; + fn encode(&mut self, value: &&str, buffer: &mut Box<[u8]>) { + let len = value.len(); + buffer[self.start..self.start + len].copy_from_slice(value.as_bytes()); + self.start += len; } } diff --git a/tests/compact_encoding.rs b/tests/compact_encoding.rs index 926868a3..6ed9ee5c 100644 --- a/tests/compact_encoding.rs +++ b/tests/compact_encoding.rs @@ -1,13 +1,26 @@ use hypercore::compact_encoding::{CompactEncoding, State}; #[test] -fn cenc_create() { +fn cenc_create() -> std::io::Result<()> { + let str_value_1 = "foo"; + let str_value_2 = "bar"; + let mut state = State::new(); assert_eq!(state.start, 0); assert_eq!(state.end, 0); - assert_eq!(state.buffer.capacity(), 0); - state.preencode("test".to_string()); + state.preencode(&str_value_1); + assert_eq!(state.start, 0); + assert_eq!(state.end, 3); + state.preencode(&str_value_2); assert_eq!(state.start, 0); - assert_eq!(state.end, 4); - assert_eq!(state.buffer.capacity(), 0); + assert_eq!(state.end, 6); + let mut buffer = state.create_buffer(); + assert_eq!(buffer.len(), 6); + state.encode(&str_value_1, &mut buffer); + assert_eq!(state.start, 3); + assert_eq!(state.end, 6); + state.encode(&str_value_2, &mut buffer); + assert_eq!(state.start, 6); + assert_eq!(state.end, 6); + Ok(()) } From bbcc95c92237bb346f912c24fd3fe53338da9a03 Mon Sep 17 00:00:00 2001 From: Timo Tiuraniemi Date: Wed, 17 Aug 2022 16:59:53 +0300 Subject: [PATCH 007/157] Some code for String, u32 and usize encoding --- src/compact_encoding/mod.rs | 187 ++++++++++++++++++++++++++++++++++-- tests/compact_encoding.rs | 29 ++---- 2 files changed, 189 insertions(+), 27 deletions(-) diff --git a/src/compact_encoding/mod.rs b/src/compact_encoding/mod.rs index c7ead70b..1ebaf48e 100644 --- a/src/compact_encoding/mod.rs +++ b/src/compact_encoding/mod.rs @@ -2,6 +2,10 @@ use std::fmt::Debug; +const U16_SIGNIFIER: u8 = 0xfd; +const U32_SIGNIFIER: u8 = 0xfe; +const U64_SIGNIFIER: u8 = 0xff; + /// State. #[derive(Debug)] pub struct State { @@ -29,11 +33,126 @@ impl State { ) } + /// Create a state from existing buffer. + pub fn from_buffer(buffer: &Box<[u8]>) -> State { + State { + start: 0, + end: buffer.len(), + } + } + /// After calling preencode(), this allocates the right size buffer to the heap. /// Follow this with the same number of encode() steps to fill the created buffer. pub fn create_buffer(&self) -> Box<[u8]> { vec![0; self.end].into_boxed_slice() } + + /// Encode u32 to 4 LE bytes. + pub fn encode_uint32(&mut self, uint: u32, buffer: &mut Box<[u8]>) { + let bytes = uint.to_le_bytes(); + buffer[self.start] = bytes[0]; + self.start += 1; + buffer[self.start] = bytes[1]; + self.start += 1; + buffer[self.start] = bytes[2]; + self.start += 1; + buffer[self.start] = bytes[3]; + self.start += 1; + } + + /// Preencode a string slice + pub fn preencode_str(&mut self, value: &str) { + self.preencode_usize_var(&value.len()); + self.end += value.len(); + } + + /// Encode a string slice + pub fn encode_str(&mut self, value: &str, buffer: &mut Box<[u8]>) { + let len = value.len(); + self.encode_usize_var(&len, buffer); + buffer[self.start..self.start + len].copy_from_slice(value.as_bytes()); + self.start += len; + } + + fn preencode_uint_var + Ord>(&mut self, uint: &T) { + self.end += if *uint <= T::from(0xfc) { + 1 + } else if *uint <= T::from(0xffff) { + 3 + } else if *uint <= T::from(0xffffffff) { + 5 + } else { + 9 + }; + } + + fn preencode_usize_var(&mut self, value: &usize) { + // TODO: This repeats the logic from above that works for u8 -> u64, but sadly not usize + self.end += if *value <= 0xfc { + 1 + } else if *value <= 0xffff { + 3 + } else if *value <= 0xffffffff { + 5 + } else { + 9 + }; + } + + fn encode_usize_var(&mut self, value: &usize, buffer: &mut Box<[u8]>) { + if *value <= 0xfc { + let bytes = value.to_le_bytes(); + buffer[self.start] = bytes[0]; + self.start += 1; + } else if *value <= 0xffff { + buffer[self.start] = U16_SIGNIFIER; + self.start += 1; + let bytes = value.to_le_bytes(); + buffer[self.start] = bytes[0]; + self.start += 1; + buffer[self.start] = bytes[1]; + self.start += 1; + } else if *value <= 0xffffffff { + buffer[self.start] = U32_SIGNIFIER; + self.start += 1; + let bytes = value.to_le_bytes(); + buffer[self.start] = bytes[0]; + self.start += 1; + buffer[self.start] = bytes[1]; + self.start += 1; + buffer[self.start] = bytes[2]; + self.start += 1; + buffer[self.start] = bytes[3]; + self.start += 1; + } else { + buffer[self.start] = U64_SIGNIFIER; + self.start += 1; + let bytes = value.to_le_bytes(); + buffer[self.start] = bytes[0]; + self.start += 1; + buffer[self.start] = bytes[1]; + self.start += 1; + buffer[self.start] = bytes[2]; + self.start += 1; + buffer[self.start] = bytes[3]; + self.start += 1; + buffer[self.start] = bytes[4]; + self.start += 1; + buffer[self.start] = bytes[5]; + self.start += 1; + buffer[self.start] = bytes[6]; + self.start += 1; + buffer[self.start] = bytes[7]; + self.start += 1; + } + } + + fn decode_usize_var(&mut self, buffer: &Box<[u8]>) -> usize { + // FIXME: need to check first byte here for signifier etc. + let value = buffer[self.start]; + self.start += 1; + value.into() + } } /// Compact Encoding @@ -47,18 +166,70 @@ where /// Encode fn encode(&mut self, value: &T, buffer: &mut Box<[u8]>); - // /// Decode - // fn decode(state: State) -> T; + /// Decode + fn decode(&mut self, buffer: &Box<[u8]>) -> T; } -impl CompactEncoding<&str> for State { - fn preencode(&mut self, value: &&str) { - self.end += value.len(); +impl CompactEncoding for State { + fn preencode(&mut self, value: &String) { + self.preencode_str(value) } - fn encode(&mut self, value: &&str, buffer: &mut Box<[u8]>) { - let len = value.len(); - buffer[self.start..self.start + len].copy_from_slice(value.as_bytes()); + fn encode(&mut self, value: &String, buffer: &mut Box<[u8]>) { + self.encode_str(value, buffer) + } + + fn decode(&mut self, buffer: &Box<[u8]>) -> String { + let len = self.decode_usize_var(buffer); + let value = std::str::from_utf8(&buffer[self.start..self.start + len]) + .expect("string is invalid UTF-8"); self.start += len; + value.to_string() + } +} + +impl CompactEncoding for State { + fn preencode(&mut self, value: &u32) { + self.preencode_uint_var(value) + } + + fn encode(&mut self, value: &u32, buffer: &mut Box<[u8]>) { + if *value <= 0xfc { + let bytes = value.to_le_bytes(); + buffer[self.start] = bytes[0]; + self.start += 1; + } else if *value <= 0xffff { + buffer[self.start] = U16_SIGNIFIER; + self.start += 1; + let bytes = value.to_le_bytes(); + buffer[self.start] = bytes[0]; + self.start += 1; + buffer[self.start] = bytes[1]; + self.start += 1; + } else { + buffer[self.start] = U32_SIGNIFIER; + self.start += 1; + self.encode_uint32(*value, buffer); + } + } + + fn decode(&mut self, _buffer: &Box<[u8]>) -> u32 { + // FIXME + 0 + } +} + +impl CompactEncoding for State { + fn preencode(&mut self, value: &usize) { + self.preencode_usize_var(value) + } + + fn encode(&mut self, value: &usize, buffer: &mut Box<[u8]>) { + self.encode_usize_var(value, buffer) + } + + fn decode(&mut self, buffer: &Box<[u8]>) -> usize { + // FIXME + buffer.len() } } diff --git a/tests/compact_encoding.rs b/tests/compact_encoding.rs index 6ed9ee5c..eedcb74e 100644 --- a/tests/compact_encoding.rs +++ b/tests/compact_encoding.rs @@ -1,26 +1,17 @@ use hypercore::compact_encoding::{CompactEncoding, State}; #[test] -fn cenc_create() -> std::io::Result<()> { +fn cenc_create() { let str_value_1 = "foo"; let str_value_2 = "bar"; - let mut state = State::new(); - assert_eq!(state.start, 0); - assert_eq!(state.end, 0); - state.preencode(&str_value_1); - assert_eq!(state.start, 0); - assert_eq!(state.end, 3); - state.preencode(&str_value_2); - assert_eq!(state.start, 0); - assert_eq!(state.end, 6); - let mut buffer = state.create_buffer(); - assert_eq!(buffer.len(), 6); - state.encode(&str_value_1, &mut buffer); - assert_eq!(state.start, 3); - assert_eq!(state.end, 6); - state.encode(&str_value_2, &mut buffer); - assert_eq!(state.start, 6); - assert_eq!(state.end, 6); - Ok(()) + let mut enc_state = State::new(); + enc_state.preencode_str(&str_value_1); + enc_state.preencode_str(&str_value_2); + let mut buffer = enc_state.create_buffer(); + enc_state.encode_str(&str_value_1, &mut buffer); + enc_state.encode_str(&str_value_2, &mut buffer); + let mut dec_state = State::from_buffer(&buffer); + let str_value_1_ret: String = dec_state.decode(&buffer); + assert_eq!(str_value_1, str_value_1_ret); } From 47e4dbc6bded4955ff8a857e4e75ab63ecffe7fe Mon Sep 17 00:00:00 2001 From: Timo Tiuraniemi Date: Thu, 18 Aug 2022 11:06:18 +0300 Subject: [PATCH 008/157] Some more work and tests --- src/compact_encoding/mod.rs | 144 ++++++++++++++++++++++-------------- tests/compact_encoding.rs | 72 +++++++++++++++++- 2 files changed, 156 insertions(+), 60 deletions(-) diff --git a/src/compact_encoding/mod.rs b/src/compact_encoding/mod.rs index 1ebaf48e..1b9936f7 100644 --- a/src/compact_encoding/mod.rs +++ b/src/compact_encoding/mod.rs @@ -1,5 +1,6 @@ //! Compact encoding module. Rust implementation of https://github.com/compact-encoding/compact-encoding. +use std::convert::TryFrom; use std::fmt::Debug; const U16_SIGNIFIER: u8 = 0xfd; @@ -49,15 +50,7 @@ impl State { /// Encode u32 to 4 LE bytes. pub fn encode_uint32(&mut self, uint: u32, buffer: &mut Box<[u8]>) { - let bytes = uint.to_le_bytes(); - buffer[self.start] = bytes[0]; - self.start += 1; - buffer[self.start] = bytes[1]; - self.start += 1; - buffer[self.start] = bytes[2]; - self.start += 1; - buffer[self.start] = bytes[3]; - self.start += 1; + self.encode_uint32_bytes(&uint.to_le_bytes(), buffer); } /// Preencode a string slice @@ -74,8 +67,62 @@ impl State { self.start += len; } + /// Decode a u16 + pub fn decode_u16(&mut self, buffer: &Box<[u8]>) -> u16 { + let value: u16 = + ((buffer[self.start] as u16) << 0) | ((buffer[self.start + 1] as u16) << 8); + self.start += 2; + value + } + + /// Decode a u32 + pub fn decode_u32(&mut self, buffer: &Box<[u8]>) -> u32 { + let value: u32 = ((buffer[self.start] as u32) << 0) + | ((buffer[self.start + 1] as u32) << 8) + | ((buffer[self.start + 2] as u32) << 16) + | ((buffer[self.start + 3] as u32) << 24); + self.start += 4; + value + } + + /// Decode a u64 + pub fn decode_u64(&mut self, buffer: &Box<[u8]>) -> u64 { + let value: u64 = ((buffer[self.start] as u64) << 0) + | ((buffer[self.start + 1] as u64) << 8) + | ((buffer[self.start + 2] as u64) << 16) + | ((buffer[self.start + 3] as u64) << 24) + | ((buffer[self.start + 4] as u64) << 32) + | ((buffer[self.start + 5] as u64) << 40) + | ((buffer[self.start + 6] as u64) << 48) + | ((buffer[self.start + 7] as u64) << 56); + self.start += 8; + value + } + + fn encode_uint16_bytes(&mut self, bytes: &[u8], buffer: &mut Box<[u8]>) { + buffer[self.start] = bytes[0]; + buffer[self.start + 1] = bytes[1]; + self.start += 2; + } + + fn encode_uint32_bytes(&mut self, bytes: &[u8], buffer: &mut Box<[u8]>) { + self.encode_uint16_bytes(bytes, buffer); + buffer[self.start] = bytes[2]; + buffer[self.start + 1] = bytes[3]; + self.start += 2; + } + + fn encode_uint64_bytes(&mut self, bytes: &[u8], buffer: &mut Box<[u8]>) { + self.encode_uint32_bytes(bytes, buffer); + buffer[self.start] = bytes[4]; + buffer[self.start + 1] = bytes[5]; + buffer[self.start + 2] = bytes[6]; + buffer[self.start + 3] = bytes[7]; + self.start += 4; + } + fn preencode_uint_var + Ord>(&mut self, uint: &T) { - self.end += if *uint <= T::from(0xfc) { + self.end += if *uint < T::from(U16_SIGNIFIER.into()) { 1 } else if *uint <= T::from(0xffff) { 3 @@ -88,7 +135,7 @@ impl State { fn preencode_usize_var(&mut self, value: &usize) { // TODO: This repeats the logic from above that works for u8 -> u64, but sadly not usize - self.end += if *value <= 0xfc { + self.end += if *value < U16_SIGNIFIER.into() { 1 } else if *value <= 0xffff { 3 @@ -107,51 +154,34 @@ impl State { } else if *value <= 0xffff { buffer[self.start] = U16_SIGNIFIER; self.start += 1; - let bytes = value.to_le_bytes(); - buffer[self.start] = bytes[0]; - self.start += 1; - buffer[self.start] = bytes[1]; - self.start += 1; + self.encode_uint16_bytes(&value.to_le_bytes(), buffer); } else if *value <= 0xffffffff { buffer[self.start] = U32_SIGNIFIER; self.start += 1; - let bytes = value.to_le_bytes(); - buffer[self.start] = bytes[0]; - self.start += 1; - buffer[self.start] = bytes[1]; - self.start += 1; - buffer[self.start] = bytes[2]; - self.start += 1; - buffer[self.start] = bytes[3]; - self.start += 1; + self.encode_uint32_bytes(&value.to_le_bytes(), buffer); } else { buffer[self.start] = U64_SIGNIFIER; self.start += 1; - let bytes = value.to_le_bytes(); - buffer[self.start] = bytes[0]; - self.start += 1; - buffer[self.start] = bytes[1]; - self.start += 1; - buffer[self.start] = bytes[2]; - self.start += 1; - buffer[self.start] = bytes[3]; - self.start += 1; - buffer[self.start] = bytes[4]; - self.start += 1; - buffer[self.start] = bytes[5]; - self.start += 1; - buffer[self.start] = bytes[6]; - self.start += 1; - buffer[self.start] = bytes[7]; - self.start += 1; + self.encode_uint64_bytes(&value.to_le_bytes(), buffer); } } fn decode_usize_var(&mut self, buffer: &Box<[u8]>) -> usize { - // FIXME: need to check first byte here for signifier etc. - let value = buffer[self.start]; + let first = buffer[self.start]; self.start += 1; - value.into() + // NB: the from_le_bytes needs a [u8; 2] and that can't be efficiently + // created from a byte slice. + if first < U16_SIGNIFIER { + first.into() + } else if first == U16_SIGNIFIER { + self.decode_u16(buffer).into() + } else if first == U32_SIGNIFIER { + usize::try_from(self.decode_u32(buffer)) + .expect("Attempted converting to a 32 bit usize on below 32 bit system") + } else { + usize::try_from(self.decode_u64(buffer)) + .expect("Attempted converting to a 64 bit usize on below 64 bit system") + } } } @@ -194,18 +224,14 @@ impl CompactEncoding for State { } fn encode(&mut self, value: &u32, buffer: &mut Box<[u8]>) { - if *value <= 0xfc { + if *value < U16_SIGNIFIER.into() { let bytes = value.to_le_bytes(); buffer[self.start] = bytes[0]; self.start += 1; } else if *value <= 0xffff { buffer[self.start] = U16_SIGNIFIER; self.start += 1; - let bytes = value.to_le_bytes(); - buffer[self.start] = bytes[0]; - self.start += 1; - buffer[self.start] = bytes[1]; - self.start += 1; + self.encode_uint16_bytes(&value.to_le_bytes(), buffer); } else { buffer[self.start] = U32_SIGNIFIER; self.start += 1; @@ -213,9 +239,16 @@ impl CompactEncoding for State { } } - fn decode(&mut self, _buffer: &Box<[u8]>) -> u32 { - // FIXME - 0 + fn decode(&mut self, buffer: &Box<[u8]>) -> u32 { + let first = buffer[self.start]; + self.start += 1; + if first < U16_SIGNIFIER { + first.into() + } else if first == U16_SIGNIFIER { + self.decode_u16(buffer).into() + } else { + self.decode_u32(buffer).into() + } } } @@ -229,7 +262,6 @@ impl CompactEncoding for State { } fn decode(&mut self, buffer: &Box<[u8]>) -> usize { - // FIXME - buffer.len() + self.decode_usize_var(buffer) } } diff --git a/tests/compact_encoding.rs b/tests/compact_encoding.rs index eedcb74e..092b6e37 100644 --- a/tests/compact_encoding.rs +++ b/tests/compact_encoding.rs @@ -1,17 +1,81 @@ use hypercore::compact_encoding::{CompactEncoding, State}; +// The max value for 1 byte length is 252 +const MAX_ONE_BYTE_UINT: u8 = 252; + +// The min value for 2 byte length is 253 +const MIN_TWO_BYTE_UINT: u8 = 253; + #[test] -fn cenc_create() { +fn cenc_basic() { let str_value_1 = "foo"; - let str_value_2 = "bar"; + let str_value_2 = (0..MAX_ONE_BYTE_UINT).map(|_| "X").collect::(); + let u32_value_3: u32 = u32::MAX; + let u32_value_4: u32 = 0xF0E1D2C3; let mut enc_state = State::new(); enc_state.preencode_str(&str_value_1); - enc_state.preencode_str(&str_value_2); + enc_state.preencode(&str_value_2); + enc_state.preencode(&u32_value_3); + enc_state.preencode(&u32_value_4); let mut buffer = enc_state.create_buffer(); + // Strings: 1 byte for length, 3/252 bytes for content + // u32: 1 byte for u32 signifier, 4 bytes for data + assert_eq!(buffer.len(), 1 + 3 + 1 + 252 + 1 + 4 + 1 + 4); enc_state.encode_str(&str_value_1, &mut buffer); - enc_state.encode_str(&str_value_2, &mut buffer); + enc_state.encode(&str_value_2, &mut buffer); + enc_state.encode(&u32_value_3, &mut buffer); + enc_state.encode(&u32_value_4, &mut buffer); let mut dec_state = State::from_buffer(&buffer); let str_value_1_ret: String = dec_state.decode(&buffer); assert_eq!(str_value_1, str_value_1_ret); + let str_value_2_ret: String = dec_state.decode(&buffer); + assert_eq!(str_value_2, str_value_2_ret); + let u32_value_3_ret: u32 = dec_state.decode(&buffer); + assert_eq!(u32_value_3, u32_value_3_ret); + let u32_value_4_ret: u32 = dec_state.decode(&buffer); + assert_eq!(u32_value_4, u32_value_4_ret); +} + +#[test] +fn cenc_string_long() { + let str_value = (0..MIN_TWO_BYTE_UINT).map(|_| "X").collect::(); + assert_eq!(str_value.len(), 253); + let mut enc_state = State::new(); + enc_state.preencode(&str_value); + let mut buffer = enc_state.create_buffer(); + // 1 byte for u16 signifier, 2 bytes for length, 256 bytes for data + assert_eq!(buffer.len(), 1 + 2 + 253); + enc_state.encode(&str_value, &mut buffer); + let mut dec_state = State::from_buffer(&buffer); + let str_value_ret: String = dec_state.decode(&buffer); + assert_eq!(str_value, str_value_ret); +} + +#[test] +fn cenc_u32_as_u16() { + let u32_value: u32 = u16::MAX.into(); + let mut enc_state = State::new(); + enc_state.preencode(&u32_value); + let mut buffer = enc_state.create_buffer(); + // 1 byte for u16 signifier, 2 bytes for length + assert_eq!(buffer.len(), 1 + 2); + enc_state.encode(&u32_value, &mut buffer); + let mut dec_state = State::from_buffer(&buffer); + let u32_value_ret: u32 = dec_state.decode(&buffer); + assert_eq!(u32_value, u32_value_ret); +} + +#[test] +fn cenc_u32_as_u8() { + let u32_value: u32 = MAX_ONE_BYTE_UINT.into(); + let mut enc_state = State::new(); + enc_state.preencode(&u32_value); + let mut buffer = enc_state.create_buffer(); + // 1 byte for data + assert_eq!(buffer.len(), 1); + enc_state.encode(&u32_value, &mut buffer); + let mut dec_state = State::from_buffer(&buffer); + let u32_value_ret: u32 = dec_state.decode(&buffer); + assert_eq!(u32_value, u32_value_ret); } From f48f84497c27304448e355b1c0fbd5606f589e1a Mon Sep 17 00:00:00 2001 From: Timo Tiuraniemi Date: Thu, 18 Aug 2022 11:42:09 +0300 Subject: [PATCH 009/157] Skeleton for Oplog --- src/feed.rs | 6 ++++++ src/feed_builder.rs | 4 ++++ src/lib.rs | 1 + src/oplog/mod.rs | 10 ++++++++++ 4 files changed, 21 insertions(+) create mode 100644 src/oplog/mod.rs diff --git a/src/feed.rs b/src/feed.rs index afa14876..3778293b 100644 --- a/src/feed.rs +++ b/src/feed.rs @@ -9,6 +9,7 @@ use crate::bitfield::Bitfield; use crate::crypto::{ generate_keypair, sign, verify, Hash, Merkle, PublicKey, SecretKey, Signature, }; +use crate::oplog::Oplog; use crate::proof::Proof; use anyhow::{bail, ensure, Result}; use flat_tree as flat; @@ -70,6 +71,8 @@ where pub(crate) length: u64, /// Bitfield to keep track of which data we own. pub(crate) bitfield: Bitfield, + /// Oplog + pub(crate) oplog: Oplog, pub(crate) tree: TreeIndex, pub(crate) peers: Vec, } @@ -414,6 +417,9 @@ where // TODO: check peers.length, call ._announce if peers exist. } + // TODO: use the oplog + self.oplog.append(); + // TODO: Discern between "primary" and "replica" streams. // if (!this.writable) { // if (!this._synced) this._synced = this.bitfield.iterator(0, this.length) diff --git a/src/feed_builder.rs b/src/feed_builder.rs index e4ac5419..c130db8e 100644 --- a/src/feed_builder.rs +++ b/src/feed_builder.rs @@ -2,6 +2,7 @@ use ed25519_dalek::{PublicKey, SecretKey}; use crate::bitfield::Bitfield; use crate::crypto::Merkle; +use crate::oplog::Oplog; use crate::storage::Storage; use random_access_storage::RandomAccess; use std::fmt::Debug; @@ -51,6 +52,8 @@ where } else { Bitfield::new() }; + // TODO: use the oplog + let oplog = Oplog {}; use crate::storage::Node; let mut tree = TreeIndex::new(tree); @@ -79,6 +82,7 @@ where byte_length, length: tree.blocks(), bitfield, + oplog, tree, public_key: self.public_key, secret_key: self.secret_key, diff --git a/src/lib.rs b/src/lib.rs index 3a822861..49281a5f 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -39,6 +39,7 @@ mod crypto; mod event; mod feed; mod feed_builder; +mod oplog; mod proof; mod replicate; mod storage; diff --git a/src/oplog/mod.rs b/src/oplog/mod.rs new file mode 100644 index 00000000..442b794c --- /dev/null +++ b/src/oplog/mod.rs @@ -0,0 +1,10 @@ +/// Operation log +#[derive(Debug)] +pub struct Oplog {} + +impl Oplog { + /// Appends an entry to the oplog + pub fn append(&self) { + // TODO + } +} From bfe10d1c982240065a0f88280398bff3c65ed732 Mon Sep 17 00:00:00 2001 From: Timo Tiuraniemi Date: Thu, 18 Aug 2022 13:45:30 +0300 Subject: [PATCH 010/157] Again asserting test --- src/feed.rs | 6 +++--- tests/js/interop.js | 11 +---------- tests/js_interop.rs | 4 ++-- 3 files changed, 6 insertions(+), 15 deletions(-) diff --git a/src/feed.rs b/src/feed.rs index 3778293b..8ac14665 100644 --- a/src/feed.rs +++ b/src/feed.rs @@ -158,6 +158,9 @@ where .write_data(self.byte_length as u64, &data) .await?; + // TODO: use the oplog + self.oplog.append(); + let hash = Hash::from_roots(self.merkle.roots()); let index = self.length; let message = hash_with_length_as_bytes(hash, index + 1); @@ -417,9 +420,6 @@ where // TODO: check peers.length, call ._announce if peers exist. } - // TODO: use the oplog - self.oplog.append(); - // TODO: Discern between "primary" and "replica" streams. // if (!this.writable) { // if (!this._synced) this._synced = this.bitfield.iterator(0, this.length) diff --git a/tests/js/interop.js b/tests/js/interop.js index dfc7cc82..f1f19988 100644 --- a/tests/js/interop.js +++ b/tests/js/interop.js @@ -33,16 +33,7 @@ if (process.argv[2] === '1') { async function step1(testSet) { let core = new Hypercore(`work/${testSet}`, testKeyPair.publicKey, {keyPair: testKeyPair}); - let len = await core.append(['Hello', 'World', 'foo', 'bar']); - console.log("LEN", len); - len = await core.append(['zan', 'tup']); - console.log("LEN2", len); - len = await core.append(['fup']); - console.log("LEN3", len); - len = await core.append(['sup']); - console.log("LEN4", len); - // len = await core.append(['flushes']); - // console.log("LEN5", len); + await core.append(['Hello', 'World']); await core.close(); core = new Hypercore(`work/${testSet}`, testKeyPair.publicKey, {keyPair: testKeyPair}); }; diff --git a/tests/js_interop.rs b/tests/js_interop.rs index 6add0da1..1814b0f8 100644 --- a/tests/js_interop.rs +++ b/tests/js_interop.rs @@ -37,7 +37,7 @@ fn js_interop_js_first() { let work_dir = prepare_test_set(TEST_SET_JS_FIRST); js_step_1_create_hypercore(TEST_SET_JS_FIRST); let hash = common::create_hypercore_hash(work_dir).expect("Could not hash directory"); - // assert_eq!(get_step_1_hash(), hash) + assert_eq!(get_step_1_hash(), hash) } #[async_std::test] @@ -46,7 +46,7 @@ async fn js_interop_rs_first() { init(); let work_dir = prepare_test_set(TEST_SET_RS_FIRST); step_1_create_hypercore(&work_dir).await; - let hash = common::create_hypercore_hash(work_dir).expect("Could not hash directory"); + let _hash = common::create_hypercore_hash(work_dir).expect("Could not hash directory"); // TODO: Make this match, only data does right now // assert_eq!(get_step_1_hash(), hash) } From 39a3df134b9e9b21aed2b63470536f3b45a2d09f Mon Sep 17 00:00:00 2001 From: Timo Tiuraniemi Date: Fri, 19 Aug 2022 11:43:18 +0300 Subject: [PATCH 011/157] Encoding of byte buffer --- src/compact_encoding/mod.rs | 21 +++++++++++++++++++++ tests/compact_encoding.rs | 14 ++++++++++++++ 2 files changed, 35 insertions(+) diff --git a/src/compact_encoding/mod.rs b/src/compact_encoding/mod.rs index 1b9936f7..481c6112 100644 --- a/src/compact_encoding/mod.rs +++ b/src/compact_encoding/mod.rs @@ -265,3 +265,24 @@ impl CompactEncoding for State { self.decode_usize_var(buffer) } } + +impl CompactEncoding> for State { + fn preencode(&mut self, value: &Box<[u8]>) { + let len = value.len(); + self.preencode_usize_var(&len); + self.end += len; + } + + fn encode(&mut self, value: &Box<[u8]>, buffer: &mut Box<[u8]>) { + let len = value.len(); + self.encode_usize_var(&len, buffer); + buffer[self.start..self.start + len].copy_from_slice(value); + self.start += len; + } + + fn decode(&mut self, buffer: &Box<[u8]>) -> Box<[u8]> { + let len = self.decode_usize_var(buffer); + let vec: Vec = buffer[self.start..self.start + len].to_vec(); + vec.into_boxed_slice() + } +} diff --git a/tests/compact_encoding.rs b/tests/compact_encoding.rs index 092b6e37..c7e98bfa 100644 --- a/tests/compact_encoding.rs +++ b/tests/compact_encoding.rs @@ -79,3 +79,17 @@ fn cenc_u32_as_u8() { let u32_value_ret: u32 = dec_state.decode(&buffer); assert_eq!(u32_value, u32_value_ret); } + +#[test] +fn cenc_buffer() { + let buf_value = vec![0xFF, 0x00].into_boxed_slice(); + let mut enc_state = State::new(); + enc_state.preencode(&buf_value); + let mut buffer = enc_state.create_buffer(); + // 1 byte for length, 2 bytes for data + assert_eq!(buffer.len(), 3); + enc_state.encode(&buf_value, &mut buffer); + let mut dec_state = State::from_buffer(&buffer); + let buf_value_ret: Box<[u8]> = dec_state.decode(&buffer); + assert_eq!(buf_value, buf_value_ret); +} From eeabc4a1fc118513087833d908fd3f29473a89a7 Mon Sep 17 00:00:00 2001 From: Timo Tiuraniemi Date: Mon, 22 Aug 2022 16:00:56 +0300 Subject: [PATCH 012/157] Initial oplog structs --- src/compact_encoding/mod.rs | 228 ++++++++++++++++++++++++++-------- src/feed.rs | 6 - src/feed_builder.rs | 5 +- src/oplog/mod.rs | 237 +++++++++++++++++++++++++++++++++++- tests/compact_encoding.rs | 44 +++++++ 5 files changed, 453 insertions(+), 67 deletions(-) diff --git a/src/compact_encoding/mod.rs b/src/compact_encoding/mod.rs index 481c6112..9731d8b2 100644 --- a/src/compact_encoding/mod.rs +++ b/src/compact_encoding/mod.rs @@ -67,7 +67,29 @@ impl State { self.start += len; } - /// Decode a u16 + /// Decode a String + pub fn decode_string(&mut self, buffer: &Box<[u8]>) -> String { + let len = self.decode_usize_var(buffer); + let value = std::str::from_utf8(&buffer[self.start..self.start + len]) + .expect("string is invalid UTF-8"); + self.start += len; + value.to_string() + } + + /// Preencode a variable length usigned int + pub fn preencode_uint_var + Ord>(&mut self, uint: &T) { + self.end += if *uint < T::from(U16_SIGNIFIER.into()) { + 1 + } else if *uint <= T::from(0xffff) { + 3 + } else if *uint <= T::from(0xffffffff) { + 5 + } else { + 9 + }; + } + + /// Decode a fixed length u16 pub fn decode_u16(&mut self, buffer: &Box<[u8]>) -> u16 { let value: u16 = ((buffer[self.start] as u16) << 0) | ((buffer[self.start + 1] as u16) << 8); @@ -75,7 +97,37 @@ impl State { value } - /// Decode a u32 + /// Encode a variable length u32 + pub fn encode_u32_var(&mut self, value: &u32, buffer: &mut Box<[u8]>) { + if *value < U16_SIGNIFIER.into() { + let bytes = value.to_le_bytes(); + buffer[self.start] = bytes[0]; + self.start += 1; + } else if *value <= 0xffff { + buffer[self.start] = U16_SIGNIFIER; + self.start += 1; + self.encode_uint16_bytes(&value.to_le_bytes(), buffer); + } else { + buffer[self.start] = U32_SIGNIFIER; + self.start += 1; + self.encode_uint32(*value, buffer); + } + } + + /// Decode a variable length u32 + pub fn decode_u32_var(&mut self, buffer: &Box<[u8]>) -> u32 { + let first = buffer[self.start]; + self.start += 1; + if first < U16_SIGNIFIER { + first.into() + } else if first == U16_SIGNIFIER { + self.decode_u16(buffer).into() + } else { + self.decode_u32(buffer).into() + } + } + + /// Decode a fixed length u32 pub fn decode_u32(&mut self, buffer: &Box<[u8]>) -> u32 { let value: u32 = ((buffer[self.start] as u32) << 0) | ((buffer[self.start + 1] as u32) << 8) @@ -85,7 +137,43 @@ impl State { value } - /// Decode a u64 + /// Encode a variable length u64 + pub fn encode_u64_var(&mut self, value: &u64, buffer: &mut Box<[u8]>) { + if *value < U16_SIGNIFIER.into() { + let bytes = value.to_le_bytes(); + buffer[self.start] = bytes[0]; + self.start += 1; + } else if *value <= 0xffff { + buffer[self.start] = U16_SIGNIFIER; + self.start += 1; + self.encode_uint16_bytes(&value.to_le_bytes(), buffer); + } else if *value <= 0xffffffff { + buffer[self.start] = U32_SIGNIFIER; + self.start += 1; + self.encode_uint32_bytes(&value.to_le_bytes(), buffer); + } else { + buffer[self.start] = U64_SIGNIFIER; + self.start += 1; + self.encode_uint64_bytes(&value.to_le_bytes(), buffer); + } + } + + /// Decode a variable length u64 + pub fn decode_u64_var(&mut self, buffer: &Box<[u8]>) -> u64 { + let first = buffer[self.start]; + self.start += 1; + if first < U16_SIGNIFIER { + first.into() + } else if first == U16_SIGNIFIER { + self.decode_u16(buffer).into() + } else if first == U32_SIGNIFIER { + self.decode_u32(buffer).into() + } else { + self.decode_u64(buffer) + } + } + + /// Decode a fixed length u64 pub fn decode_u64(&mut self, buffer: &Box<[u8]>) -> u64 { let value: u64 = ((buffer[self.start] as u64) << 0) | ((buffer[self.start + 1] as u64) << 8) @@ -99,6 +187,57 @@ impl State { value } + /// Preencode a byte buffer + pub fn preencode_buffer(&mut self, value: &Box<[u8]>) { + let len = value.len(); + self.preencode_usize_var(&len); + self.end += len; + } + + /// Encode a byte buffer + pub fn encode_buffer(&mut self, value: &Box<[u8]>, buffer: &mut Box<[u8]>) { + let len = value.len(); + self.encode_usize_var(&len, buffer); + buffer[self.start..self.start + len].copy_from_slice(value); + self.start += len; + } + + /// Decode a byte buffer + pub fn decode_buffer(&mut self, buffer: &Box<[u8]>) -> Box<[u8]> { + let len = self.decode_usize_var(buffer); + buffer[self.start..self.start + len] + .to_vec() + .into_boxed_slice() + } + + /// Preencode a string array + pub fn preencode_string_array(&mut self, value: &Vec) { + let len = value.len(); + self.preencode_usize_var(&len); + for string_value in value.into_iter() { + self.preencode_str(string_value); + } + } + + /// Encode a String array + pub fn encode_string_array(&mut self, value: &Vec, buffer: &mut Box<[u8]>) { + let len = value.len(); + self.encode_usize_var(&len, buffer); + for string_value in value { + self.encode_str(string_value, buffer); + } + } + + /// Decode a String array + pub fn decode_string_array(&mut self, buffer: &Box<[u8]>) -> Vec { + let len = self.decode_usize_var(buffer); + let mut value = Vec::with_capacity(len); + for _ in 0..len { + value.push(self.decode_string(buffer)); + } + value + } + fn encode_uint16_bytes(&mut self, bytes: &[u8], buffer: &mut Box<[u8]>) { buffer[self.start] = bytes[0]; buffer[self.start + 1] = bytes[1]; @@ -121,18 +260,6 @@ impl State { self.start += 4; } - fn preencode_uint_var + Ord>(&mut self, uint: &T) { - self.end += if *uint < T::from(U16_SIGNIFIER.into()) { - 1 - } else if *uint <= T::from(0xffff) { - 3 - } else if *uint <= T::from(0xffffffff) { - 5 - } else { - 9 - }; - } - fn preencode_usize_var(&mut self, value: &usize) { // TODO: This repeats the logic from above that works for u8 -> u64, but sadly not usize self.end += if *value < U16_SIGNIFIER.into() { @@ -210,11 +337,7 @@ impl CompactEncoding for State { } fn decode(&mut self, buffer: &Box<[u8]>) -> String { - let len = self.decode_usize_var(buffer); - let value = std::str::from_utf8(&buffer[self.start..self.start + len]) - .expect("string is invalid UTF-8"); - self.start += len; - value.to_string() + self.decode_string(buffer) } } @@ -224,31 +347,25 @@ impl CompactEncoding for State { } fn encode(&mut self, value: &u32, buffer: &mut Box<[u8]>) { - if *value < U16_SIGNIFIER.into() { - let bytes = value.to_le_bytes(); - buffer[self.start] = bytes[0]; - self.start += 1; - } else if *value <= 0xffff { - buffer[self.start] = U16_SIGNIFIER; - self.start += 1; - self.encode_uint16_bytes(&value.to_le_bytes(), buffer); - } else { - buffer[self.start] = U32_SIGNIFIER; - self.start += 1; - self.encode_uint32(*value, buffer); - } + self.encode_u32_var(value, buffer) } fn decode(&mut self, buffer: &Box<[u8]>) -> u32 { - let first = buffer[self.start]; - self.start += 1; - if first < U16_SIGNIFIER { - first.into() - } else if first == U16_SIGNIFIER { - self.decode_u16(buffer).into() - } else { - self.decode_u32(buffer).into() - } + self.decode_u32_var(buffer) + } +} + +impl CompactEncoding for State { + fn preencode(&mut self, value: &u64) { + self.preencode_uint_var(value) + } + + fn encode(&mut self, value: &u64, buffer: &mut Box<[u8]>) { + self.encode_u64_var(value, buffer) + } + + fn decode(&mut self, buffer: &Box<[u8]>) -> u64 { + self.decode_u64_var(buffer) } } @@ -268,21 +385,28 @@ impl CompactEncoding for State { impl CompactEncoding> for State { fn preencode(&mut self, value: &Box<[u8]>) { - let len = value.len(); - self.preencode_usize_var(&len); - self.end += len; + self.preencode_buffer(value); } fn encode(&mut self, value: &Box<[u8]>, buffer: &mut Box<[u8]>) { - let len = value.len(); - self.encode_usize_var(&len, buffer); - buffer[self.start..self.start + len].copy_from_slice(value); - self.start += len; + self.encode_buffer(value, buffer); } fn decode(&mut self, buffer: &Box<[u8]>) -> Box<[u8]> { - let len = self.decode_usize_var(buffer); - let vec: Vec = buffer[self.start..self.start + len].to_vec(); - vec.into_boxed_slice() + self.decode_buffer(buffer) + } +} + +impl CompactEncoding> for State { + fn preencode(&mut self, value: &Vec) { + self.preencode_string_array(value); + } + + fn encode(&mut self, value: &Vec, buffer: &mut Box<[u8]>) { + self.encode_string_array(value, buffer); + } + + fn decode(&mut self, buffer: &Box<[u8]>) -> Vec { + self.decode_string_array(buffer) } } diff --git a/src/feed.rs b/src/feed.rs index 8ac14665..afa14876 100644 --- a/src/feed.rs +++ b/src/feed.rs @@ -9,7 +9,6 @@ use crate::bitfield::Bitfield; use crate::crypto::{ generate_keypair, sign, verify, Hash, Merkle, PublicKey, SecretKey, Signature, }; -use crate::oplog::Oplog; use crate::proof::Proof; use anyhow::{bail, ensure, Result}; use flat_tree as flat; @@ -71,8 +70,6 @@ where pub(crate) length: u64, /// Bitfield to keep track of which data we own. pub(crate) bitfield: Bitfield, - /// Oplog - pub(crate) oplog: Oplog, pub(crate) tree: TreeIndex, pub(crate) peers: Vec, } @@ -158,9 +155,6 @@ where .write_data(self.byte_length as u64, &data) .await?; - // TODO: use the oplog - self.oplog.append(); - let hash = Hash::from_roots(self.merkle.roots()); let index = self.length; let message = hash_with_length_as_bytes(hash, index + 1); diff --git a/src/feed_builder.rs b/src/feed_builder.rs index c130db8e..60744e43 100644 --- a/src/feed_builder.rs +++ b/src/feed_builder.rs @@ -2,7 +2,6 @@ use ed25519_dalek::{PublicKey, SecretKey}; use crate::bitfield::Bitfield; use crate::crypto::Merkle; -use crate::oplog::Oplog; use crate::storage::Storage; use random_access_storage::RandomAccess; use std::fmt::Debug; @@ -52,8 +51,7 @@ where } else { Bitfield::new() }; - // TODO: use the oplog - let oplog = Oplog {}; + use crate::storage::Node; let mut tree = TreeIndex::new(tree); @@ -82,7 +80,6 @@ where byte_length, length: tree.blocks(), bitfield, - oplog, tree, public_key: self.public_key, secret_key: self.secret_key, diff --git a/src/oplog/mod.rs b/src/oplog/mod.rs index 442b794c..de636f5f 100644 --- a/src/oplog/mod.rs +++ b/src/oplog/mod.rs @@ -1,10 +1,237 @@ -/// Operation log +use crate::compact_encoding::{CompactEncoding, State}; +use crate::crypto::{generate_keypair, PublicKey, SecretKey}; + +/// Oplog header +#[derive(Debug)] +struct Header { + types: HeaderTypes, + tree: HeaderTree, + signer: HeaderSigner, + hints: HeaderHints, + contiguous_length: u64, +} + +impl Header { + /// Creates a new Header from given public and secret keys + pub fn new_from_keys(public_key: PublicKey, secret_key: SecretKey) -> Header { + Header { + types: HeaderTypes { + tree: "blake2b".to_string(), + bitfield: "raw".to_string(), + signer: "ed25519".to_string(), + }, + tree: HeaderTree { fork: 0, length: 0 }, + signer: HeaderSigner { + public_key, + secret_key, + }, + hints: HeaderHints { reorgs: vec![] }, + contiguous_length: 0, + } + // Javascript side, initial header + // + // header = { + // types: { tree: 'blake2b', bitfield: 'raw', signer: 'ed25519' }, + // userData: [], + // tree: { + // fork: 0, + // length: 0, + // rootHash: null, + // signature: null + // }, + // signer: opts.keyPair || crypto.keyPair(), + // hints: { + // reorgs: [] + // }, + // contiguousLength: 0 + // } + } + + /// Creates a new Header by generating a key pair + pub fn new() -> Header { + let key_pair = generate_keypair(); + Header::new_from_keys(key_pair.public, key_pair.secret) + } +} + +/// Oplog header types #[derive(Debug)] -pub struct Oplog {} +struct HeaderTypes { + tree: String, + bitfield: String, + signer: String, +} + +impl CompactEncoding for State { + fn preencode(&mut self, value: &HeaderTypes) { + self.preencode(&value.tree); + self.preencode(&value.bitfield); + self.preencode(&value.signer); + } + + fn encode(&mut self, value: &HeaderTypes, buffer: &mut Box<[u8]>) { + self.encode(&value.tree, buffer); + self.encode(&value.bitfield, buffer); + self.encode(&value.signer, buffer); + } + + fn decode(&mut self, buffer: &Box<[u8]>) -> HeaderTypes { + let tree = self.decode(buffer); + let bitfield = self.decode(buffer); + let signer = self.decode(buffer); + HeaderTypes { + tree, + bitfield, + signer, + } + } +} + +/// Oplog header tree +#[derive(Debug)] +struct HeaderTree { + fork: u64, + length: u64, +} + +impl CompactEncoding for State { + fn preencode(&mut self, value: &HeaderTree) { + self.preencode(&value.fork); + self.preencode(&value.length); + } + + fn encode(&mut self, value: &HeaderTree, buffer: &mut Box<[u8]>) { + self.encode(&value.fork, buffer); + self.encode(&value.length, buffer); + } + + fn decode(&mut self, buffer: &Box<[u8]>) -> HeaderTree { + let fork = self.decode(buffer); + let length = self.decode(buffer); + HeaderTree { fork, length } + } +} + +/// Oplog header signer +#[derive(Debug)] +struct HeaderSigner { + public_key: PublicKey, + secret_key: SecretKey, +} + +impl CompactEncoding for State { + fn preencode(&mut self, value: &HeaderSigner) { + let public_key_bytes: Box<[u8]> = value.public_key.as_bytes().to_vec().into_boxed_slice(); + let secret_key_bytes: Box<[u8]> = value.secret_key.as_bytes().to_vec().into_boxed_slice(); + self.preencode(&public_key_bytes); + self.preencode(&secret_key_bytes); + } + + fn encode(&mut self, value: &HeaderSigner, buffer: &mut Box<[u8]>) { + let public_key_bytes: Box<[u8]> = value.public_key.as_bytes().to_vec().into_boxed_slice(); + let secret_key_bytes: Box<[u8]> = value.secret_key.as_bytes().to_vec().into_boxed_slice(); + self.encode(&public_key_bytes, buffer); + self.encode(&secret_key_bytes, buffer); + } + + fn decode(&mut self, buffer: &Box<[u8]>) -> HeaderSigner { + let public_key_bytes: Box<[u8]> = self.decode(buffer); + let secret_key_bytes: Box<[u8]> = self.decode(buffer); + + HeaderSigner { + public_key: PublicKey::from_bytes(&public_key_bytes).unwrap(), + secret_key: SecretKey::from_bytes(&secret_key_bytes).unwrap(), + } + } +} + +/// Oplog header hints +#[derive(Debug)] +struct HeaderHints { + reorgs: Vec, +} + +impl CompactEncoding for State { + fn preencode(&mut self, value: &HeaderHints) { + self.preencode(&value.reorgs); + } + + fn encode(&mut self, value: &HeaderHints, buffer: &mut Box<[u8]>) { + self.encode(&value.reorgs, buffer); + } + + fn decode(&mut self, buffer: &Box<[u8]>) -> HeaderHints { + HeaderHints { + reorgs: self.decode(buffer), + } + } +} + +impl CompactEncoding
for State { + fn preencode(&mut self, value: &Header) { + self.start += 1; // Version + self.preencode(&value.types); + // TODO self.preencode(&value.user_data); + self.preencode(&value.tree); + self.preencode(&value.signer); + self.preencode(&value.hints); + self.preencode(&value.contiguous_length); + } + + fn encode(&mut self, value: &Header, buffer: &mut Box<[u8]>) { + buffer[0] = 0; // Version + self.start += 1; + self.encode(&value.types, buffer); + // TODO self.encode(&value.user_data, buffer); + self.encode(&value.tree, buffer); + self.encode(&value.signer, buffer); + self.encode(&value.hints, buffer); + self.encode(&value.contiguous_length, buffer); + } + + fn decode(&mut self, buffer: &Box<[u8]>) -> Header { + let version: usize = self.decode(buffer); + if version != 0 { + panic!("Unknown oplog version {}", version); + } + let types: HeaderTypes = self.decode(buffer); + // TODO: let user_data: HeaderUserData = self.decode(buffer); + let tree: HeaderTree = self.decode(buffer); + let signer: HeaderSigner = self.decode(buffer); + let hints: HeaderHints = self.decode(buffer); + let contiguous_length: u64 = self.decode(buffer); + + Header { + types, + tree, + signer, + hints, + contiguous_length, + } + } +} + +/// Oplog +#[derive(Debug)] +pub struct Oplog { + #[allow(dead_code)] + header: Header, +} impl Oplog { - /// Appends an entry to the oplog - pub fn append(&self) { - // TODO + /// Creates a new Oplog from given public and secret keys + #[allow(dead_code)] + pub fn new_from_keys(public_key: PublicKey, secret_key: SecretKey) -> Oplog { + Oplog { + header: Header::new_from_keys(public_key, secret_key), + } + } + + /// Creates a new Oplog and generates random keys + #[allow(dead_code)] + pub fn new() -> Oplog { + Oplog { + header: Header::new(), + } } } diff --git a/tests/compact_encoding.rs b/tests/compact_encoding.rs index c7e98bfa..f7870ee1 100644 --- a/tests/compact_encoding.rs +++ b/tests/compact_encoding.rs @@ -80,6 +80,34 @@ fn cenc_u32_as_u8() { assert_eq!(u32_value, u32_value_ret); } +#[test] +fn cenc_u64() { + let u64_value: u64 = 0xF0E1D2C3B4A59687; + let mut enc_state = State::new(); + enc_state.preencode(&u64_value); + let mut buffer = enc_state.create_buffer(); + // 1 byte for u64 signifier, 8 bytes for length + assert_eq!(buffer.len(), 1 + 8); + enc_state.encode(&u64_value, &mut buffer); + let mut dec_state = State::from_buffer(&buffer); + let u64_value_ret: u64 = dec_state.decode(&buffer); + assert_eq!(u64_value, u64_value_ret); +} + +#[test] +fn cenc_u64_as_u32() { + let u64_value: u64 = u32::MAX.into(); + let mut enc_state = State::new(); + enc_state.preencode(&u64_value); + let mut buffer = enc_state.create_buffer(); + // 1 byte for u32 signifier, 4 bytes for length + assert_eq!(buffer.len(), 1 + 4); + enc_state.encode(&u64_value, &mut buffer); + let mut dec_state = State::from_buffer(&buffer); + let u64_value_ret: u64 = dec_state.decode(&buffer); + assert_eq!(u64_value, u64_value_ret); +} + #[test] fn cenc_buffer() { let buf_value = vec![0xFF, 0x00].into_boxed_slice(); @@ -93,3 +121,19 @@ fn cenc_buffer() { let buf_value_ret: Box<[u8]> = dec_state.decode(&buffer); assert_eq!(buf_value, buf_value_ret); } + +#[test] +fn cenc_string_array() { + let string_array_value = vec!["first".to_string(), "second".to_string()]; + let mut enc_state = State::new(); + enc_state.preencode(&string_array_value); + let mut buffer = enc_state.create_buffer(); + // 1 byte for array length, + // 1 byte for string length, 5 bytes for string, + // 1 byte for string length, 6 bytes for string + assert_eq!(buffer.len(), 1 + 1 + 5 + 1 + 6); + enc_state.encode(&string_array_value, &mut buffer); + let mut dec_state = State::from_buffer(&buffer); + let string_array_value_ret: Vec = dec_state.decode(&buffer); + assert_eq!(string_array_value, string_array_value_ret); +} From 9805234a6b5a5344617a4fb3bcaf27ddadb251f5 Mon Sep 17 00:00:00 2001 From: Timo Tiuraniemi Date: Mon, 22 Aug 2022 17:42:49 +0300 Subject: [PATCH 013/157] Secret key is optional --- src/oplog/mod.rs | 38 +++++++++++++++++++++++++++++--------- 1 file changed, 29 insertions(+), 9 deletions(-) diff --git a/src/oplog/mod.rs b/src/oplog/mod.rs index de636f5f..1062eda7 100644 --- a/src/oplog/mod.rs +++ b/src/oplog/mod.rs @@ -13,7 +13,7 @@ struct Header { impl Header { /// Creates a new Header from given public and secret keys - pub fn new_from_keys(public_key: PublicKey, secret_key: SecretKey) -> Header { + pub fn new_from_keys(public_key: PublicKey, secret_key: Option) -> Header { Header { types: HeaderTypes { tree: "blake2b".to_string(), @@ -50,7 +50,7 @@ impl Header { /// Creates a new Header by generating a key pair pub fn new() -> Header { let key_pair = generate_keypair(); - Header::new_from_keys(key_pair.public, key_pair.secret) + Header::new_from_keys(key_pair.public, Some(key_pair.secret)) } } @@ -116,31 +116,51 @@ impl CompactEncoding for State { #[derive(Debug)] struct HeaderSigner { public_key: PublicKey, - secret_key: SecretKey, + secret_key: Option, } impl CompactEncoding for State { fn preencode(&mut self, value: &HeaderSigner) { let public_key_bytes: Box<[u8]> = value.public_key.as_bytes().to_vec().into_boxed_slice(); - let secret_key_bytes: Box<[u8]> = value.secret_key.as_bytes().to_vec().into_boxed_slice(); self.preencode(&public_key_bytes); - self.preencode(&secret_key_bytes); + match &value.secret_key { + Some(secret_key) => { + let secret_key_bytes: Box<[u8]> = secret_key.as_bytes().to_vec().into_boxed_slice(); + self.preencode(&secret_key_bytes); + } + None => { + self.end += 1; + } + } } fn encode(&mut self, value: &HeaderSigner, buffer: &mut Box<[u8]>) { let public_key_bytes: Box<[u8]> = value.public_key.as_bytes().to_vec().into_boxed_slice(); - let secret_key_bytes: Box<[u8]> = value.secret_key.as_bytes().to_vec().into_boxed_slice(); self.encode(&public_key_bytes, buffer); - self.encode(&secret_key_bytes, buffer); + match &value.secret_key { + Some(secret_key) => { + let secret_key_bytes: Box<[u8]> = secret_key.as_bytes().to_vec().into_boxed_slice(); + self.encode(&secret_key_bytes, buffer); + } + None => { + buffer[self.start] = 0; + self.start += 1; + } + } } fn decode(&mut self, buffer: &Box<[u8]>) -> HeaderSigner { let public_key_bytes: Box<[u8]> = self.decode(buffer); let secret_key_bytes: Box<[u8]> = self.decode(buffer); + let secret_key: Option = if secret_key_bytes.len() == 0 { + None + } else { + Some(SecretKey::from_bytes(&secret_key_bytes).unwrap()) + }; HeaderSigner { public_key: PublicKey::from_bytes(&public_key_bytes).unwrap(), - secret_key: SecretKey::from_bytes(&secret_key_bytes).unwrap(), + secret_key, } } } @@ -221,7 +241,7 @@ pub struct Oplog { impl Oplog { /// Creates a new Oplog from given public and secret keys #[allow(dead_code)] - pub fn new_from_keys(public_key: PublicKey, secret_key: SecretKey) -> Oplog { + pub fn new_from_keys(public_key: PublicKey, secret_key: Option) -> Oplog { Oplog { header: Header::new_from_keys(public_key, secret_key), } From 4df7f3bd70e7166b0d65dd058998a4616b324971 Mon Sep 17 00:00:00 2001 From: Timo Tiuraniemi Date: Tue, 23 Aug 2022 10:09:28 +0300 Subject: [PATCH 014/157] Feature flag v10 code --- Cargo.toml | 3 +++ examples/async.rs | 9 +++++++++ examples/main.rs | 8 ++++++++ src/lib.rs | 17 ++++++++++++++++- src/prelude.rs | 2 ++ src/storage/mod.rs | 29 ++++++++++++++++++++++++++++- tests/common/mod.rs | 2 ++ tests/compat.rs | 8 ++++++++ tests/feed.rs | 18 ++++++++++++++++++ tests/js_interop.rs | 12 ++++++------ tests/model.rs | 2 ++ tests/regression.rs | 2 ++ tests/storage.rs | 7 +++++++ 13 files changed, 111 insertions(+), 8 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 0f18f936..218cd942 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -48,6 +48,9 @@ async-std = { version = "1.5.0", features = ["attributes"] } sha2 = "0.10.2" [features] +default = ["v10"] +# v10 version of the crate, without this, v9 is built +v10 = [] # Used only in interoperability tests under tests/js-interop which use the javascript version of hypercore # to verify that this crate works. To run them, use: # cargo test --features js-interop-tests diff --git a/examples/async.rs b/examples/async.rs index a052bfe2..14564346 100644 --- a/examples/async.rs +++ b/examples/async.rs @@ -1,8 +1,10 @@ use async_std::task; +#[cfg(not(feature = "v10"))] use hypercore::Feed; use random_access_storage::RandomAccess; use std::fmt::Debug; +#[cfg(not(feature = "v10"))] async fn append(feed: &mut Feed, content: &[u8]) where T: RandomAccess> + Debug + Send, @@ -10,6 +12,7 @@ where feed.append(content).await.unwrap(); } +#[cfg(not(feature = "v10"))] async fn print(feed: &mut Feed) where T: RandomAccess> + Debug + Send, @@ -18,6 +21,7 @@ where println!("{:?}", feed.get(1).await); } +#[cfg(not(feature = "v10"))] fn main() { task::block_on(task::spawn(async { let mut feed = Feed::default(); @@ -27,3 +31,8 @@ fn main() { print(&mut feed).await; })); } + +#[cfg(feature = "v10")] +fn main() { + unimplemented!() +} diff --git a/examples/main.rs b/examples/main.rs index d711ecc6..8acda765 100644 --- a/examples/main.rs +++ b/examples/main.rs @@ -1,6 +1,8 @@ +#[cfg(not(feature = "v10"))] use hypercore::Feed; #[async_std::main] +#[cfg(not(feature = "v10"))] async fn main() { let mut feed = Feed::open("feed.db").await.expect("Failed to create dir"); @@ -20,6 +22,12 @@ async fn main() { println!("{:?}", format_res(feed.get(3).await)); // prints "back" } +#[async_std::main] +#[cfg(feature = "v10")] +async fn main() { + unimplemented!(); +} + fn format_res(res: anyhow::Result>>) -> String { match res { Ok(Some(bytes)) => String::from_utf8(bytes).expect("Shouldnt fail in example"), diff --git a/src/lib.rs b/src/lib.rs index 49281a5f..0f719eab 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -2,7 +2,8 @@ #![forbid(rust_2018_idioms, rust_2018_compatibility)] #![forbid(missing_debug_implementations)] #![forbid(missing_docs)] -#![cfg_attr(test, deny(warnings))] +// FIXME: Off during v10 coding +// #![cfg_attr(test, deny(warnings))] //! ## Introduction //! Hypercore is a secure, distributed append-only log. Built for sharing @@ -12,6 +13,7 @@ //! //! ## Example //! ```rust +//! #[cfg(not(feature = "v10"))] //! # fn main() -> Result<(), Box> { //! # async_std::task::block_on(async { //! let mut feed = hypercore::open("./feed.db").await?; @@ -24,6 +26,13 @@ //! # Ok(()) //! # }) //! # } +//! #[cfg(feature = "v10")] +//! # fn main() -> Result<(), Box> { +//! # async_std::task::block_on(async { +//! // unimplemented +//! Ok(()) +//! # }) +//! # } //! ``` //! //! [dat-node]: https://github.com/mafintosh/hypercore @@ -37,8 +46,11 @@ pub mod prelude; mod audit; mod crypto; mod event; +#[cfg(not(feature = "v10"))] mod feed; +#[cfg(not(feature = "v10"))] mod feed_builder; +#[cfg(feature = "v10")] mod oplog; mod proof; mod replicate; @@ -46,7 +58,9 @@ mod storage; pub use crate::crypto::{generate_keypair, sign, verify, Signature}; pub use crate::event::Event; +#[cfg(not(feature = "v10"))] pub use crate::feed::Feed; +#[cfg(not(feature = "v10"))] pub use crate::feed_builder::FeedBuilder; pub use crate::proof::Proof; pub use crate::replicate::Peer; @@ -56,6 +70,7 @@ pub use ed25519_dalek::{PublicKey, SecretKey}; use std::path::Path; /// Create a new Hypercore `Feed`. +#[cfg(not(feature = "v10"))] pub async fn open>( path: P, ) -> anyhow::Result> { diff --git a/src/prelude.rs b/src/prelude.rs index eeede9a9..2519a807 100644 --- a/src/prelude.rs +++ b/src/prelude.rs @@ -2,8 +2,10 @@ //! //! ```rust //! use hypercore::prelude::*; +//! #[cfg(not(feature = "v10"))] //! let feed = Feed::default(); //! ``` +#[cfg(not(feature = "v10"))] pub use crate::feed::Feed; // pub use feed_builder::FeedBuilder; pub use crate::storage::{Node, NodeTrait, Storage, Store}; diff --git a/src/storage/mod.rs b/src/storage/mod.rs index 4ee6752a..7eb48950 100644 --- a/src/storage/mod.rs +++ b/src/storage/mod.rs @@ -39,10 +39,13 @@ pub enum Store { /// Bitfield Bitfield, /// Signatures + #[cfg(not(feature = "v10"))] Signatures, /// Keypair + #[cfg(not(feature = "v10"))] Keypair, /// Oplog + #[cfg(feature = "v10")] Oplog, } @@ -55,8 +58,11 @@ where tree: T, data: T, bitfield: T, + #[cfg(not(feature = "v10"))] signatures: T, + #[cfg(not(feature = "v10"))] keypair: T, + #[cfg(feature = "v10")] oplog: T, } @@ -75,20 +81,29 @@ where let tree = create(Store::Tree).await?; let data = create(Store::Data).await?; let bitfield = create(Store::Bitfield).await?; + #[cfg(not(feature = "v10"))] let signatures = create(Store::Signatures).await?; + #[cfg(not(feature = "v10"))] let keypair = create(Store::Keypair).await?; + + #[cfg(feature = "v10")] let oplog = create(Store::Oplog).await?; + let mut instance = Self { tree, data, bitfield, + #[cfg(not(feature = "v10"))] signatures, + #[cfg(not(feature = "v10"))] keypair, + #[cfg(feature = "v10")] oplog, }; + #[cfg(feature = "v10")] if overwrite || instance.bitfield.len().await.unwrap_or(0) == 0 { - // TODO: This is + // TODO: This has nothing in it instance .oplog .write(0, &[0x00]) @@ -105,6 +120,7 @@ where .map_err(|e| anyhow!(e))?; } + #[cfg(not(feature = "v10"))] if overwrite || instance.signatures.len().await.unwrap_or(0) == 0 { let header = create_signatures(); instance @@ -172,6 +188,7 @@ where } /// Search the signature stores for a `Signature`, starting at `index`. + #[cfg(not(feature = "v10"))] pub fn next_signature( &mut self, index: u64, @@ -195,6 +212,7 @@ where /// Get a `Signature` from the store. #[inline] + #[cfg(not(feature = "v10"))] pub async fn get_signature(&mut self, index: u64) -> Result { let bytes = self .signatures @@ -209,6 +227,7 @@ where /// TODO: Ensure the signature size is correct. /// NOTE: Should we create a `Signature` entry type? #[inline] + #[cfg(not(feature = "v10"))] pub async fn put_signature( &mut self, index: u64, @@ -336,6 +355,7 @@ where } /// Read a public key from storage + #[cfg(not(feature = "v10"))] pub async fn read_public_key(&mut self) -> Result { let buf = self .keypair @@ -347,6 +367,7 @@ where } /// Read a secret key from storage + #[cfg(not(feature = "v10"))] pub async fn read_secret_key(&mut self) -> Result { let buf = self .keypair @@ -358,12 +379,14 @@ where } /// Write a public key to the storage + #[cfg(not(feature = "v10"))] pub async fn write_public_key(&mut self, public_key: &PublicKey) -> Result<()> { let buf: [u8; PUBLIC_KEY_LENGTH] = public_key.to_bytes(); self.keypair.write(0, &buf).await.map_err(|e| anyhow!(e)) } /// Write a secret key to the storage + #[cfg(not(feature = "v10"))] pub async fn write_secret_key(&mut self, secret_key: &SecretKey) -> Result<()> { let buf: [u8; SECRET_KEY_LENGTH] = secret_key.to_bytes(); self.keypair @@ -373,6 +396,7 @@ where } /// Tries to read a partial keypair (ie: with an optional secret_key) from the storage + #[cfg(not(feature = "v10"))] pub async fn read_partial_keypair(&mut self) -> Option { match self.read_public_key().await { Ok(public) => match self.read_secret_key().await { @@ -406,8 +430,11 @@ impl Storage { Store::Tree => "tree", Store::Data => "data", Store::Bitfield => "bitfield", + #[cfg(not(feature = "v10"))] Store::Signatures => "signatures", + #[cfg(not(feature = "v10"))] Store::Keypair => "key", + #[cfg(feature = "v10")] Store::Oplog => "oplog", }; RandomAccessDisk::open(dir.as_path().join(name)).boxed() diff --git a/tests/common/mod.rs b/tests/common/mod.rs index 64aa44c1..6342d2a9 100644 --- a/tests/common/mod.rs +++ b/tests/common/mod.rs @@ -2,10 +2,12 @@ use hypercore; use anyhow::Error; use futures::future::FutureExt; +#[cfg(not(feature = "v10"))] use hypercore::{Feed, Storage, Store}; use random_access_memory as ram; use sha2::{Digest, Sha256}; +#[cfg(not(feature = "v10"))] pub async fn create_feed(page_size: usize) -> Result, Error> { let create = |_store: Store| async move { Ok(ram::RandomAccessMemory::new(page_size)) }.boxed(); let storage = Storage::new(create, false).await?; diff --git a/tests/compat.rs b/tests/compat.rs index e68c4f6d..41de7a78 100644 --- a/tests/compat.rs +++ b/tests/compat.rs @@ -11,12 +11,14 @@ use std::path::{Path, PathBuf}; use data_encoding::HEXLOWER; use ed25519_dalek::{Keypair, Signature}; +#[cfg(not(feature = "v10"))] use hypercore::Feed; use hypercore::{Storage, Store}; use random_access_disk::RandomAccessDisk; use remove_dir_all::remove_dir_all; #[async_std::test] +#[cfg(not(feature = "v10"))] async fn deterministic_data_and_tree() { let expected_tree = hex_bytes(concat!( "0502570200002807424c414b4532620000000000000000000000000000000000ab27d45f509274", @@ -57,6 +59,7 @@ fn deterministic_data_and_tree_after_replication() { } #[async_std::test] +#[cfg(not(feature = "v10"))] async fn deterministic_signatures() { let key = hex_bytes("9718a1ff1c4ca79feac551c0c7212a65e4091278ec886b88be01ee4039682238"); let keypair_bytes = hex_bytes(concat!( @@ -135,6 +138,7 @@ fn hex_bytes(hex: &str) -> Vec { HEXLOWER.decode(hex.as_bytes()).unwrap() } +#[cfg(not(feature = "v10"))] fn storage_path>(dir: P, s: Store) -> PathBuf { let filename = match s { Store::Tree => "tree", @@ -142,11 +146,13 @@ fn storage_path>(dir: P, s: Store) -> PathBuf { Store::Bitfield => "bitfield", Store::Signatures => "signatures", Store::Keypair => "key", + #[cfg(feature = "v10")] Store::Oplog => "oplog", }; dir.as_ref().join(filename) } +#[cfg(not(feature = "v10"))] async fn mk_storage() -> (PathBuf, Storage) { let temp_dir = tempfile::tempdir().unwrap(); let dir = temp_dir.into_path(); @@ -162,6 +168,7 @@ async fn mk_storage() -> (PathBuf, Storage) { (dir, storage) } +#[cfg(not(feature = "v10"))] fn read_bytes>(dir: P, s: Store) -> Vec { let mut f = File::open(storage_path(dir, s)).unwrap(); let mut bytes = Vec::new(); @@ -169,6 +176,7 @@ fn read_bytes>(dir: P, s: Store) -> Vec { bytes } +#[cfg(not(feature = "v10"))] fn mk_keypair(keypair_bytes: &[u8], public_key: &[u8]) -> Keypair { let keypair = Keypair::from_bytes(&keypair_bytes).unwrap(); assert_eq!( diff --git a/tests/feed.rs b/tests/feed.rs index 3bb1828e..5952011f 100644 --- a/tests/feed.rs +++ b/tests/feed.rs @@ -2,7 +2,9 @@ extern crate random_access_memory as ram; mod common; +#[cfg(not(feature = "v10"))] use common::create_feed; +#[cfg(not(feature = "v10"))] use hypercore::{generate_keypair, Feed, NodeTrait, PublicKey, SecretKey, Storage}; use random_access_storage::RandomAccess; use std::env::temp_dir; @@ -11,6 +13,7 @@ use std::fs; use std::io::Write; #[async_std::test] +#[cfg(not(feature = "v10"))] async fn create_with_key() { let keypair = generate_keypair(); let storage = Storage::new_memory().await.unwrap(); @@ -22,6 +25,7 @@ async fn create_with_key() { } #[async_std::test] +#[cfg(not(feature = "v10"))] async fn display() { let feed = create_feed(50).await.unwrap(); let output = format!("{}", feed); @@ -29,6 +33,7 @@ async fn display() { } #[async_std::test] +#[cfg(not(feature = "v10"))] /// Verify `.append()` and `.get()` work. async fn set_get() { let mut feed = create_feed(50).await.unwrap(); @@ -40,6 +45,7 @@ async fn set_get() { } #[async_std::test] +#[cfg(not(feature = "v10"))] async fn append() { let mut feed = create_feed(50).await.unwrap(); feed.append(br#"{"hello":"world"}"#).await.unwrap(); @@ -64,6 +70,7 @@ async fn append() { } #[async_std::test] +#[cfg(not(feature = "v10"))] /// Verify the `.root_hashes()` method returns the right nodes. async fn root_hashes() { // If no roots exist we should get an error. @@ -91,6 +98,7 @@ async fn root_hashes() { } #[async_std::test] +#[cfg(not(feature = "v10"))] async fn verify() { let mut feed = create_feed(50).await.unwrap(); let (public, secret) = copy_keys(&feed); @@ -127,6 +135,7 @@ async fn verify() { } #[async_std::test] +#[cfg(not(feature = "v10"))] async fn put() { let mut a = create_feed(50).await.unwrap(); let (public, secret) = copy_keys(&a); @@ -156,6 +165,7 @@ async fn put() { } #[async_std::test] +#[cfg(not(feature = "v10"))] /// Put data from one feed into another, while veryfing hashes. /// I.e. manual replication between two feeds. async fn put_with_data() { @@ -196,6 +206,7 @@ async fn put_with_data() { } #[async_std::test] +#[cfg(not(feature = "v10"))] async fn create_with_storage() { let storage = Storage::new_memory().await.unwrap(); assert!( @@ -205,6 +216,7 @@ async fn create_with_storage() { } #[async_std::test] +#[cfg(not(feature = "v10"))] async fn create_with_stored_public_key() { let mut storage = Storage::new_memory().await.unwrap(); let keypair = generate_keypair(); @@ -216,6 +228,7 @@ async fn create_with_stored_public_key() { } #[async_std::test] +#[cfg(not(feature = "v10"))] async fn create_with_stored_keys() { let mut storage = Storage::new_memory().await.unwrap(); let keypair = generate_keypair(); @@ -227,6 +240,7 @@ async fn create_with_stored_keys() { ); } +#[cfg(not(feature = "v10"))] fn copy_keys( feed: &Feed> + Debug + Send>, ) -> (PublicKey, SecretKey) { @@ -245,6 +259,7 @@ fn copy_keys( } #[async_std::test] +#[cfg(not(feature = "v10"))] async fn audit() { let mut feed = create_feed(50).await.unwrap(); feed.append(b"hello").await.unwrap(); @@ -261,6 +276,7 @@ async fn audit() { } #[async_std::test] +#[cfg(not(feature = "v10"))] async fn audit_bad_data() { let mut dir = temp_dir(); dir.push("audit_bad_data"); @@ -310,6 +326,7 @@ async fn audit_bad_data() { } #[async_std::test] +#[cfg(not(feature = "v10"))] async fn try_open_missing_dir() { use rand::distributions::Alphanumeric; use rand::{thread_rng, Rng}; @@ -333,6 +350,7 @@ async fn try_open_missing_dir() { } #[async_std::test] +#[cfg(not(feature = "v10"))] async fn try_open_file_as_dir() { if Feed::open("Cargo.toml").await.is_ok() { panic!("Opening path that points to a file must result in error"); diff --git a/tests/js_interop.rs b/tests/js_interop.rs index 1814b0f8..1ae9d532 100644 --- a/tests/js_interop.rs +++ b/tests/js_interop.rs @@ -3,7 +3,7 @@ mod js; use std::{path::Path, sync::Once}; use ed25519_dalek::{PublicKey, SecretKey, PUBLIC_KEY_LENGTH, SECRET_KEY_LENGTH}; -use hypercore::{FeedBuilder, Storage}; +use hypercore::Storage; use js::{cleanup, install, js_step_1_create_hypercore, prepare_test_set}; const TEST_SET_JS_FIRST: &str = "jsfirst"; @@ -58,12 +58,12 @@ async fn step_1_create_hypercore(work_dir: &str) { let public_key = PublicKey::from_bytes(&TEST_PUBLIC_KEY_BYTES).unwrap(); let secret_key = SecretKey::from_bytes(&TEST_SECRET_KEY_BYTES).unwrap(); - let builder = FeedBuilder::new(public_key, storage); - let mut feed = builder.secret_key(secret_key).build().await.unwrap(); + // let builder = FeedBuilder::new(public_key, storage); + // let mut feed = builder.secret_key(secret_key).build().await.unwrap(); - feed.append(b"Hello").await.unwrap(); - feed.append(b"World").await.unwrap(); - drop(feed); + // feed.append(b"Hello").await.unwrap(); + // feed.append(b"World").await.unwrap(); + // drop(feed); } fn get_step_1_hash() -> common::HypercoreHash { diff --git a/tests/model.rs b/tests/model.rs index 967f58ba..27648b8e 100644 --- a/tests/model.rs +++ b/tests/model.rs @@ -1,5 +1,6 @@ mod common; +#[cfg(not(feature = "v10"))] use common::create_feed; use quickcheck::{quickcheck, Arbitrary, Gen}; use rand::seq::SliceRandom; @@ -37,6 +38,7 @@ impl Arbitrary for Op { } } +#[cfg(not(feature = "v10"))] quickcheck! { fn implementation_matches_model(ops: Vec) -> bool { async_std::task::block_on(async { diff --git a/tests/regression.rs b/tests/regression.rs index 8b6a00e6..61cf60ef 100644 --- a/tests/regression.rs +++ b/tests/regression.rs @@ -1,5 +1,6 @@ mod common; +#[cfg(not(feature = "v10"))] use common::create_feed; // Postmortem: errors were happening correctly, but the error check in @@ -7,6 +8,7 @@ use common::create_feed; // checking inclusively `<=`. All we had to do was fix the check, and we all // good. #[async_std::test] +#[cfg(not(feature = "v10"))] async fn regression_01() { let mut feed = create_feed(50).await.unwrap(); assert_eq!(feed.len(), 0); diff --git a/tests/storage.rs b/tests/storage.rs index d540b310..834bab32 100644 --- a/tests/storage.rs +++ b/tests/storage.rs @@ -1,7 +1,11 @@ +#[cfg(not(feature = "v10"))] use ed25519_dalek::PublicKey; + +#[cfg(not(feature = "v10"))] use hypercore::{generate_keypair, sign, verify, Signature, Storage}; #[async_std::test] +#[cfg(not(feature = "v10"))] async fn should_write_and_read_keypair() { let keypair = generate_keypair(); let msg = b"hello"; @@ -25,6 +29,7 @@ async fn should_write_and_read_keypair() { } #[async_std::test] +#[cfg(not(feature = "v10"))] async fn should_read_partial_keypair() { let keypair = generate_keypair(); let mut storage = Storage::new_memory().await.unwrap(); @@ -38,6 +43,7 @@ async fn should_read_partial_keypair() { } #[async_std::test] +#[cfg(not(feature = "v10"))] async fn should_read_no_keypair() { let mut storage = Storage::new_memory().await.unwrap(); let partial = storage.read_partial_keypair().await; @@ -45,6 +51,7 @@ async fn should_read_no_keypair() { } #[async_std::test] +#[cfg(not(feature = "v10"))] async fn should_read_empty_public_key() { let mut storage = Storage::new_memory().await.unwrap(); assert!(storage.read_public_key().await.is_err()); From 4cfe1c4575014be334286011c8094cbcf8645403 Mon Sep 17 00:00:00 2001 From: Timo Tiuraniemi Date: Tue, 23 Aug 2022 10:55:48 +0300 Subject: [PATCH 015/157] Move random-access-disk to dev deps in v10 to make WASM build possible --- Cargo.toml | 4 +++- src/storage/mod.rs | 6 ++---- tests/common/mod.rs | 28 ++++++++++++++++++++++++++-- tests/js_interop.rs | 3 ++- 4 files changed, 33 insertions(+), 8 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 218cd942..fdcc245e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -28,7 +28,7 @@ memory-pager = "0.9.0" merkle-tree-stream = "0.12.0" pretty-hash = "0.4.1" rand = "0.7.3" -random-access-disk = "2.0.0" +random-access-disk = { version = "2.0.0", optional = true } random-access-memory = "2.0.0" random-access-storage = "4.0.0" sha2 = "0.9.2" @@ -46,11 +46,13 @@ remove_dir_all = "0.7.0" tempfile = "3.1.0" async-std = { version = "1.5.0", features = ["attributes"] } sha2 = "0.10.2" +random-access-disk = "2.0.0" [features] default = ["v10"] # v10 version of the crate, without this, v9 is built v10 = [] +v9 = ["dep:random-access-disk"] # Used only in interoperability tests under tests/js-interop which use the javascript version of hypercore # to verify that this crate works. To run them, use: # cargo test --features js-interop-tests diff --git a/src/storage/mod.rs b/src/storage/mod.rs index 7eb48950..7e15cbe4 100644 --- a/src/storage/mod.rs +++ b/src/storage/mod.rs @@ -11,6 +11,7 @@ use anyhow::{anyhow, ensure, Result}; use ed25519_dalek::{PublicKey, SecretKey, Signature, PUBLIC_KEY_LENGTH, SECRET_KEY_LENGTH}; use flat_tree as flat; use futures::future::FutureExt; +#[cfg(not(feature = "v10"))] use random_access_disk::RandomAccessDisk; use random_access_memory::RandomAccessMemory; use random_access_storage::RandomAccess; @@ -422,6 +423,7 @@ impl Storage { } } +#[cfg(not(feature = "v10"))] impl Storage { /// Create a new instance backed by a `RandomAccessDisk` instance. pub async fn new_disk(dir: &PathBuf, overwrite: bool) -> Result { @@ -430,12 +432,8 @@ impl Storage { Store::Tree => "tree", Store::Data => "data", Store::Bitfield => "bitfield", - #[cfg(not(feature = "v10"))] Store::Signatures => "signatures", - #[cfg(not(feature = "v10"))] Store::Keypair => "key", - #[cfg(feature = "v10")] - Store::Oplog => "oplog", }; RandomAccessDisk::open(dir.as_path().join(name)).boxed() }; diff --git a/tests/common/mod.rs b/tests/common/mod.rs index 6342d2a9..eff2a4fa 100644 --- a/tests/common/mod.rs +++ b/tests/common/mod.rs @@ -1,11 +1,14 @@ use hypercore; -use anyhow::Error; +use anyhow::{Error, Result}; use futures::future::FutureExt; #[cfg(not(feature = "v10"))] -use hypercore::{Feed, Storage, Store}; +use hypercore::Feed; +use hypercore::{Storage, Store}; +use random_access_disk::RandomAccessDisk; use random_access_memory as ram; use sha2::{Digest, Sha256}; +use std::path::PathBuf; #[cfg(not(feature = "v10"))] pub async fn create_feed(page_size: usize) -> Result, Error> { @@ -43,3 +46,24 @@ pub fn hash_file(file: String) -> Result { let hash_bytes = hasher.finalize(); Ok(format!("{:X}", hash_bytes)) } + +pub async fn create_disk_storage( + dir: &PathBuf, + overwrite: bool, +) -> Result> { + let storage = |storage: Store| { + let name = match storage { + Store::Tree => "tree", + Store::Data => "data", + Store::Bitfield => "bitfield", + #[cfg(not(feature = "v10"))] + Store::Signatures => "signatures", + #[cfg(not(feature = "v10"))] + Store::Keypair => "key", + #[cfg(feature = "v10")] + Store::Oplog => "oplog", + }; + RandomAccessDisk::open(dir.as_path().join(name)).boxed() + }; + Ok(Storage::new(storage, overwrite).await?) +} diff --git a/tests/js_interop.rs b/tests/js_interop.rs index 1ae9d532..62f761f1 100644 --- a/tests/js_interop.rs +++ b/tests/js_interop.rs @@ -2,6 +2,7 @@ mod common; mod js; use std::{path::Path, sync::Once}; +use common::create_disk_storage; use ed25519_dalek::{PublicKey, SecretKey, PUBLIC_KEY_LENGTH, SECRET_KEY_LENGTH}; use hypercore::Storage; use js::{cleanup, install, js_step_1_create_hypercore, prepare_test_set}; @@ -53,7 +54,7 @@ async fn js_interop_rs_first() { async fn step_1_create_hypercore(work_dir: &str) { let path = Path::new(work_dir).to_owned(); - let storage = Storage::new_disk(&path, false).await.unwrap(); + let storage = create_disk_storage(&path, false).await.unwrap(); let public_key = PublicKey::from_bytes(&TEST_PUBLIC_KEY_BYTES).unwrap(); let secret_key = SecretKey::from_bytes(&TEST_SECRET_KEY_BYTES).unwrap(); From 4cd4f12fb9903703b37db6a38cda065ce90d36fb Mon Sep 17 00:00:00 2001 From: Timo Tiuraniemi Date: Tue, 23 Aug 2022 14:31:03 +0300 Subject: [PATCH 016/157] Revert "Move random-access-disk to dev deps in v10 to make WASM build possible" This reverts commit 4cfe1c4575014be334286011c8094cbcf8645403. --- Cargo.toml | 4 +--- src/storage/mod.rs | 6 ++++-- tests/common/mod.rs | 28 ++-------------------------- tests/js_interop.rs | 3 +-- 4 files changed, 8 insertions(+), 33 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index fdcc245e..218cd942 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -28,7 +28,7 @@ memory-pager = "0.9.0" merkle-tree-stream = "0.12.0" pretty-hash = "0.4.1" rand = "0.7.3" -random-access-disk = { version = "2.0.0", optional = true } +random-access-disk = "2.0.0" random-access-memory = "2.0.0" random-access-storage = "4.0.0" sha2 = "0.9.2" @@ -46,13 +46,11 @@ remove_dir_all = "0.7.0" tempfile = "3.1.0" async-std = { version = "1.5.0", features = ["attributes"] } sha2 = "0.10.2" -random-access-disk = "2.0.0" [features] default = ["v10"] # v10 version of the crate, without this, v9 is built v10 = [] -v9 = ["dep:random-access-disk"] # Used only in interoperability tests under tests/js-interop which use the javascript version of hypercore # to verify that this crate works. To run them, use: # cargo test --features js-interop-tests diff --git a/src/storage/mod.rs b/src/storage/mod.rs index 7e15cbe4..7eb48950 100644 --- a/src/storage/mod.rs +++ b/src/storage/mod.rs @@ -11,7 +11,6 @@ use anyhow::{anyhow, ensure, Result}; use ed25519_dalek::{PublicKey, SecretKey, Signature, PUBLIC_KEY_LENGTH, SECRET_KEY_LENGTH}; use flat_tree as flat; use futures::future::FutureExt; -#[cfg(not(feature = "v10"))] use random_access_disk::RandomAccessDisk; use random_access_memory::RandomAccessMemory; use random_access_storage::RandomAccess; @@ -423,7 +422,6 @@ impl Storage { } } -#[cfg(not(feature = "v10"))] impl Storage { /// Create a new instance backed by a `RandomAccessDisk` instance. pub async fn new_disk(dir: &PathBuf, overwrite: bool) -> Result { @@ -432,8 +430,12 @@ impl Storage { Store::Tree => "tree", Store::Data => "data", Store::Bitfield => "bitfield", + #[cfg(not(feature = "v10"))] Store::Signatures => "signatures", + #[cfg(not(feature = "v10"))] Store::Keypair => "key", + #[cfg(feature = "v10")] + Store::Oplog => "oplog", }; RandomAccessDisk::open(dir.as_path().join(name)).boxed() }; diff --git a/tests/common/mod.rs b/tests/common/mod.rs index eff2a4fa..6342d2a9 100644 --- a/tests/common/mod.rs +++ b/tests/common/mod.rs @@ -1,14 +1,11 @@ use hypercore; -use anyhow::{Error, Result}; +use anyhow::Error; use futures::future::FutureExt; #[cfg(not(feature = "v10"))] -use hypercore::Feed; -use hypercore::{Storage, Store}; -use random_access_disk::RandomAccessDisk; +use hypercore::{Feed, Storage, Store}; use random_access_memory as ram; use sha2::{Digest, Sha256}; -use std::path::PathBuf; #[cfg(not(feature = "v10"))] pub async fn create_feed(page_size: usize) -> Result, Error> { @@ -46,24 +43,3 @@ pub fn hash_file(file: String) -> Result { let hash_bytes = hasher.finalize(); Ok(format!("{:X}", hash_bytes)) } - -pub async fn create_disk_storage( - dir: &PathBuf, - overwrite: bool, -) -> Result> { - let storage = |storage: Store| { - let name = match storage { - Store::Tree => "tree", - Store::Data => "data", - Store::Bitfield => "bitfield", - #[cfg(not(feature = "v10"))] - Store::Signatures => "signatures", - #[cfg(not(feature = "v10"))] - Store::Keypair => "key", - #[cfg(feature = "v10")] - Store::Oplog => "oplog", - }; - RandomAccessDisk::open(dir.as_path().join(name)).boxed() - }; - Ok(Storage::new(storage, overwrite).await?) -} diff --git a/tests/js_interop.rs b/tests/js_interop.rs index 62f761f1..1ae9d532 100644 --- a/tests/js_interop.rs +++ b/tests/js_interop.rs @@ -2,7 +2,6 @@ mod common; mod js; use std::{path::Path, sync::Once}; -use common::create_disk_storage; use ed25519_dalek::{PublicKey, SecretKey, PUBLIC_KEY_LENGTH, SECRET_KEY_LENGTH}; use hypercore::Storage; use js::{cleanup, install, js_step_1_create_hypercore, prepare_test_set}; @@ -54,7 +53,7 @@ async fn js_interop_rs_first() { async fn step_1_create_hypercore(work_dir: &str) { let path = Path::new(work_dir).to_owned(); - let storage = create_disk_storage(&path, false).await.unwrap(); + let storage = Storage::new_disk(&path, false).await.unwrap(); let public_key = PublicKey::from_bytes(&TEST_PUBLIC_KEY_BYTES).unwrap(); let secret_key = SecretKey::from_bytes(&TEST_SECRET_KEY_BYTES).unwrap(); From 04e0db2619c5708812436d4716331073d0f727f4 Mon Sep 17 00:00:00 2001 From: Timo Tiuraniemi Date: Tue, 23 Aug 2022 14:34:39 +0300 Subject: [PATCH 017/157] Make v10 buildable with WASM --- Cargo.toml | 4 +++- src/storage/mod.rs | 2 ++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index 218cd942..c2cea5d7 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -28,7 +28,6 @@ memory-pager = "0.9.0" merkle-tree-stream = "0.12.0" pretty-hash = "0.4.1" rand = "0.7.3" -random-access-disk = "2.0.0" random-access-memory = "2.0.0" random-access-storage = "4.0.0" sha2 = "0.9.2" @@ -39,6 +38,9 @@ bitfield-rle = "0.2.0" futures = "0.3.4" async-std = "1.5.0" +[target.'cfg(not(target_arch = "wasm32"))'.dependencies] +random-access-disk = "2.0.0" + [dev-dependencies] quickcheck = "0.9.2" data-encoding = "2.2.0" diff --git a/src/storage/mod.rs b/src/storage/mod.rs index 7eb48950..becc3c8f 100644 --- a/src/storage/mod.rs +++ b/src/storage/mod.rs @@ -11,6 +11,7 @@ use anyhow::{anyhow, ensure, Result}; use ed25519_dalek::{PublicKey, SecretKey, Signature, PUBLIC_KEY_LENGTH, SECRET_KEY_LENGTH}; use flat_tree as flat; use futures::future::FutureExt; +#[cfg(not(target_arch = "wasm32"))] use random_access_disk::RandomAccessDisk; use random_access_memory::RandomAccessMemory; use random_access_storage::RandomAccess; @@ -422,6 +423,7 @@ impl Storage { } } +#[cfg(not(target_arch = "wasm32"))] impl Storage { /// Create a new instance backed by a `RandomAccessDisk` instance. pub async fn new_disk(dir: &PathBuf, overwrite: bool) -> Result { From e1b168bb6b42427259f6b1e957fa444a901853b1 Mon Sep 17 00:00:00 2001 From: Timo Tiuraniemi Date: Wed, 24 Aug 2022 13:01:53 +0300 Subject: [PATCH 018/157] Hypercore is the type for the v10 interface, some work to make that happen --- src/hypercore.rs | 59 +++++++++++++++++++++++++++++++++++++ src/lib.rs | 6 +++- src/oplog/mod.rs | 71 +++++++++++++++++---------------------------- src/storage/mod.rs | 37 ++++++++++++++++++----- tests/common/mod.rs | 21 +++++++++++++- tests/hypercore.rs | 23 +++++++++++++++ tests/js_interop.rs | 24 +++------------ 7 files changed, 167 insertions(+), 74 deletions(-) create mode 100644 src/hypercore.rs create mode 100644 tests/hypercore.rs diff --git a/src/hypercore.rs b/src/hypercore.rs new file mode 100644 index 00000000..b04f0c5d --- /dev/null +++ b/src/hypercore.rs @@ -0,0 +1,59 @@ +//! Hypercore's main abstraction. Exposes an append-only, secure log structure. + +pub use crate::storage::{PartialKeypair, Storage}; + +use crate::{ + crypto::{generate_keypair, PublicKey, SecretKey}, + oplog::Oplog, +}; +use anyhow::Result; +use random_access_storage::RandomAccess; +use std::fmt::Debug; + +/// Hypercore is an append-only log structure. +#[derive(Debug)] +pub struct Hypercore +where + T: RandomAccess> + Debug, +{ + pub(crate) key_pair: PartialKeypair, + pub(crate) storage: Storage, + pub(crate) oplog: Oplog, + // /// Merkle tree instance. + // pub(crate) tree: Merkle, + // /// Bitfield to keep track of which data we own. + // pub(crate) bitfield: Bitfield, +} + +impl Hypercore +where + T: RandomAccess> + Debug + Send, +{ + /// Creates new hypercore using given storage with random key pair + pub async fn new(storage: Storage) -> Result> { + let key_pair = generate_keypair(); + Hypercore::new_with_key_pair( + storage, + PartialKeypair { + public: key_pair.public, + secret: Some(key_pair.secret), + }, + ) + .await + } + + /// Creates new hypercore with given storage and (partial) key pair + pub async fn new_with_key_pair( + mut storage: Storage, + key_pair: PartialKeypair, + ) -> Result> { + let oplog_bytes = storage.read_oplog().await?; + let oplog = Oplog::open(key_pair.clone(), oplog_bytes); + + Ok(Hypercore { + key_pair, + storage, + oplog, + }) + } +} diff --git a/src/lib.rs b/src/lib.rs index 0f719eab..4c7db915 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -51,6 +51,8 @@ mod feed; #[cfg(not(feature = "v10"))] mod feed_builder; #[cfg(feature = "v10")] +mod hypercore; +#[cfg(feature = "v10")] mod oplog; mod proof; mod replicate; @@ -64,8 +66,10 @@ pub use crate::feed::Feed; pub use crate::feed_builder::FeedBuilder; pub use crate::proof::Proof; pub use crate::replicate::Peer; -pub use crate::storage::{Node, NodeTrait, Storage, Store}; +pub use crate::storage::{Node, NodeTrait, PartialKeypair, Storage, Store}; pub use ed25519_dalek::{PublicKey, SecretKey}; +#[cfg(feature = "v10")] +pub use hypercore::Hypercore; use std::path::Path; diff --git a/src/oplog/mod.rs b/src/oplog/mod.rs index 1062eda7..0ed21cb9 100644 --- a/src/oplog/mod.rs +++ b/src/oplog/mod.rs @@ -1,19 +1,20 @@ use crate::compact_encoding::{CompactEncoding, State}; use crate::crypto::{generate_keypair, PublicKey, SecretKey}; +use crate::PartialKeypair; /// Oplog header #[derive(Debug)] struct Header { types: HeaderTypes, tree: HeaderTree, - signer: HeaderSigner, + signer: PartialKeypair, hints: HeaderHints, contiguous_length: u64, } impl Header { - /// Creates a new Header from given public and secret keys - pub fn new_from_keys(public_key: PublicKey, secret_key: Option) -> Header { + /// Creates a new Header from given key pair + pub fn new(key_pair: PartialKeypair) -> Header { Header { types: HeaderTypes { tree: "blake2b".to_string(), @@ -21,10 +22,7 @@ impl Header { signer: "ed25519".to_string(), }, tree: HeaderTree { fork: 0, length: 0 }, - signer: HeaderSigner { - public_key, - secret_key, - }, + signer: key_pair, hints: HeaderHints { reorgs: vec![] }, contiguous_length: 0, } @@ -46,12 +44,6 @@ impl Header { // contiguousLength: 0 // } } - - /// Creates a new Header by generating a key pair - pub fn new() -> Header { - let key_pair = generate_keypair(); - Header::new_from_keys(key_pair.public, Some(key_pair.secret)) - } } /// Oplog header types @@ -112,18 +104,11 @@ impl CompactEncoding for State { } } -/// Oplog header signer -#[derive(Debug)] -struct HeaderSigner { - public_key: PublicKey, - secret_key: Option, -} - -impl CompactEncoding for State { - fn preencode(&mut self, value: &HeaderSigner) { - let public_key_bytes: Box<[u8]> = value.public_key.as_bytes().to_vec().into_boxed_slice(); +impl CompactEncoding for State { + fn preencode(&mut self, value: &PartialKeypair) { + let public_key_bytes: Box<[u8]> = value.public.as_bytes().to_vec().into_boxed_slice(); self.preencode(&public_key_bytes); - match &value.secret_key { + match &value.secret { Some(secret_key) => { let secret_key_bytes: Box<[u8]> = secret_key.as_bytes().to_vec().into_boxed_slice(); self.preencode(&secret_key_bytes); @@ -134,10 +119,10 @@ impl CompactEncoding for State { } } - fn encode(&mut self, value: &HeaderSigner, buffer: &mut Box<[u8]>) { - let public_key_bytes: Box<[u8]> = value.public_key.as_bytes().to_vec().into_boxed_slice(); + fn encode(&mut self, value: &PartialKeypair, buffer: &mut Box<[u8]>) { + let public_key_bytes: Box<[u8]> = value.public.as_bytes().to_vec().into_boxed_slice(); self.encode(&public_key_bytes, buffer); - match &value.secret_key { + match &value.secret { Some(secret_key) => { let secret_key_bytes: Box<[u8]> = secret_key.as_bytes().to_vec().into_boxed_slice(); self.encode(&secret_key_bytes, buffer); @@ -149,18 +134,18 @@ impl CompactEncoding for State { } } - fn decode(&mut self, buffer: &Box<[u8]>) -> HeaderSigner { + fn decode(&mut self, buffer: &Box<[u8]>) -> PartialKeypair { let public_key_bytes: Box<[u8]> = self.decode(buffer); let secret_key_bytes: Box<[u8]> = self.decode(buffer); - let secret_key: Option = if secret_key_bytes.len() == 0 { + let secret: Option = if secret_key_bytes.len() == 0 { None } else { Some(SecretKey::from_bytes(&secret_key_bytes).unwrap()) }; - HeaderSigner { - public_key: PublicKey::from_bytes(&public_key_bytes).unwrap(), - secret_key, + PartialKeypair { + public: PublicKey::from_bytes(&public_key_bytes).unwrap(), + secret, } } } @@ -217,7 +202,7 @@ impl CompactEncoding
for State { let types: HeaderTypes = self.decode(buffer); // TODO: let user_data: HeaderUserData = self.decode(buffer); let tree: HeaderTree = self.decode(buffer); - let signer: HeaderSigner = self.decode(buffer); + let signer: PartialKeypair = self.decode(buffer); let hints: HeaderHints = self.decode(buffer); let contiguous_length: u64 = self.decode(buffer); @@ -239,19 +224,15 @@ pub struct Oplog { } impl Oplog { - /// Creates a new Oplog from given public and secret keys + /// Opens an new Oplog from given key pair and existing content as a byte buffer #[allow(dead_code)] - pub fn new_from_keys(public_key: PublicKey, secret_key: Option) -> Oplog { - Oplog { - header: Header::new_from_keys(public_key, secret_key), - } - } - - /// Creates a new Oplog and generates random keys - #[allow(dead_code)] - pub fn new() -> Oplog { - Oplog { - header: Header::new(), + pub fn open(key_pair: PartialKeypair, existing: Box<[u8]>) -> Oplog { + if existing.len() == 0 { + Oplog { + header: Header::new(key_pair), + } + } else { + unimplemented!("Reading an exising oplog is not supported yet"); } } } diff --git a/src/storage/mod.rs b/src/storage/mod.rs index becc3c8f..92f9b88c 100644 --- a/src/storage/mod.rs +++ b/src/storage/mod.rs @@ -15,7 +15,7 @@ use futures::future::FutureExt; use random_access_disk::RandomAccessDisk; use random_access_memory::RandomAccessMemory; use random_access_storage::RandomAccess; -use sleep_parser::*; +use sleep_parser::{create_bitfield, create_signatures, create_tree, Header}; use std::borrow::Borrow; use std::convert::TryFrom; use std::fmt::Debug; @@ -24,12 +24,31 @@ use std::path::PathBuf; const HEADER_OFFSET: u64 = 32; +/// Key pair where for read-only hypercores the secret key can also be missing. #[derive(Debug)] pub struct PartialKeypair { + /// Public key pub public: PublicKey, + /// Secret key. If None, the hypercore is read-only. pub secret: Option, } +impl Clone for PartialKeypair { + fn clone(&self) -> Self { + let secret: Option = match &self.secret { + Some(secret) => { + let bytes = secret.to_bytes(); + Some(SecretKey::from_bytes(&bytes).unwrap()) + } + None => None, + }; + PartialKeypair { + public: self.public.clone(), + secret, + } + } +} + /// The types of stores that can be created. #[derive(Debug)] pub enum Store { @@ -104,12 +123,8 @@ where #[cfg(feature = "v10")] if overwrite || instance.bitfield.len().await.unwrap_or(0) == 0 { - // TODO: This has nothing in it - instance - .oplog - .write(0, &[0x00]) - .await - .map_err(|e| anyhow!(e))?; + // Init the oplog as an empty file + instance.oplog.write(0, &[]).await.map_err(|e| anyhow!(e))?; } if overwrite || instance.bitfield.len().await.unwrap_or(0) == 0 { @@ -355,6 +370,14 @@ where } } + /// Read the full oplog bytes. + #[cfg(feature = "v10")] + pub async fn read_oplog(&mut self) -> Result> { + let len = self.oplog.len().await.map_err(|e| anyhow!(e))?; + let buf = self.oplog.read(0, len).await.map_err(|e| anyhow!(e))?; + Ok(buf.into_boxed_slice()) + } + /// Read a public key from storage #[cfg(not(feature = "v10"))] pub async fn read_public_key(&mut self) -> Result { diff --git a/tests/common/mod.rs b/tests/common/mod.rs index 6342d2a9..e775ce4d 100644 --- a/tests/common/mod.rs +++ b/tests/common/mod.rs @@ -1,12 +1,25 @@ -use hypercore; +use hypercore::{self, PartialKeypair}; use anyhow::Error; +use ed25519_dalek::{PublicKey, SecretKey, PUBLIC_KEY_LENGTH, SECRET_KEY_LENGTH}; use futures::future::FutureExt; #[cfg(not(feature = "v10"))] use hypercore::{Feed, Storage, Store}; use random_access_memory as ram; use sha2::{Digest, Sha256}; +const TEST_PUBLIC_KEY_BYTES: [u8; PUBLIC_KEY_LENGTH] = [ + 0x97, 0x60, 0x6c, 0xaa, 0xd2, 0xb0, 0x8c, 0x1d, 0x5f, 0xe1, 0x64, 0x2e, 0xee, 0xa5, 0x62, 0xcb, + 0x91, 0xd6, 0x55, 0xe2, 0x00, 0xc8, 0xd4, 0x3a, 0x32, 0x09, 0x1d, 0x06, 0x4a, 0x33, 0x1e, 0xe3, +]; +// NB: In the javascript version this is 64 bytes, but that's because sodium appends the the public +// key after the secret key for some reason. Only the first 32 bytes are actually used in +// javascript side too for signing. +const TEST_SECRET_KEY_BYTES: [u8; SECRET_KEY_LENGTH] = [ + 0x27, 0xe6, 0x74, 0x25, 0xc1, 0xff, 0xd1, 0xd9, 0xee, 0x62, 0x5c, 0x96, 0x2b, 0x57, 0x13, 0xc3, + 0x51, 0x0b, 0x71, 0x14, 0x15, 0xf3, 0x31, 0xf6, 0xfa, 0x9e, 0xf2, 0xbf, 0x23, 0x5f, 0x2f, 0xfe, +]; + #[cfg(not(feature = "v10"))] pub async fn create_feed(page_size: usize) -> Result, Error> { let create = |_store: Store| async move { Ok(ram::RandomAccessMemory::new(page_size)) }.boxed(); @@ -22,6 +35,12 @@ pub struct HypercoreHash { pub tree: String, } +pub fn get_test_key_pair() -> PartialKeypair { + let public = PublicKey::from_bytes(&TEST_PUBLIC_KEY_BYTES).unwrap(); + let secret = Some(SecretKey::from_bytes(&TEST_SECRET_KEY_BYTES).unwrap()); + PartialKeypair { public, secret } +} + pub fn create_hypercore_hash(dir: String) -> Result { let bitfield = hash_file(format!("{}/bitfield", dir))?; let data = hash_file(format!("{}/data", dir))?; diff --git a/tests/hypercore.rs b/tests/hypercore.rs new file mode 100644 index 00000000..09e9bf36 --- /dev/null +++ b/tests/hypercore.rs @@ -0,0 +1,23 @@ +mod common; + +use anyhow::Result; +use common::get_test_key_pair; +#[cfg(feature = "v10")] +use hypercore::{Hypercore, Storage}; + +#[async_std::test] +#[cfg(feature = "v10")] +async fn hypercore_new() -> Result<()> { + let storage = Storage::new_memory().await?; + let _hypercore = Hypercore::new(storage).await?; + Ok(()) +} + +#[async_std::test] +#[cfg(feature = "v10")] +async fn hypercore_new_with_key_pair() -> Result<()> { + let storage = Storage::new_memory().await?; + let key_pair = get_test_key_pair(); + let _hypercore = Hypercore::new_with_key_pair(storage, key_pair).await?; + Ok(()) +} diff --git a/tests/js_interop.rs b/tests/js_interop.rs index 1ae9d532..abd28f04 100644 --- a/tests/js_interop.rs +++ b/tests/js_interop.rs @@ -2,25 +2,12 @@ mod common; mod js; use std::{path::Path, sync::Once}; -use ed25519_dalek::{PublicKey, SecretKey, PUBLIC_KEY_LENGTH, SECRET_KEY_LENGTH}; -use hypercore::Storage; +use common::{create_hypercore_hash, get_test_key_pair}; use js::{cleanup, install, js_step_1_create_hypercore, prepare_test_set}; const TEST_SET_JS_FIRST: &str = "jsfirst"; const TEST_SET_RS_FIRST: &str = "rsfirst"; -const TEST_PUBLIC_KEY_BYTES: [u8; PUBLIC_KEY_LENGTH] = [ - 0x97, 0x60, 0x6c, 0xaa, 0xd2, 0xb0, 0x8c, 0x1d, 0x5f, 0xe1, 0x64, 0x2e, 0xee, 0xa5, 0x62, 0xcb, - 0x91, 0xd6, 0x55, 0xe2, 0x00, 0xc8, 0xd4, 0x3a, 0x32, 0x09, 0x1d, 0x06, 0x4a, 0x33, 0x1e, 0xe3, -]; -// NB: In the javascript version this is 64 bytes, but that's because sodium appends the the public -// key after the secret key for some reason. Only the first 32 bytes are actually used in -// javascript side too for signing. -const TEST_SECRET_KEY_BYTES: [u8; SECRET_KEY_LENGTH] = [ - 0x27, 0xe6, 0x74, 0x25, 0xc1, 0xff, 0xd1, 0xd9, 0xee, 0x62, 0x5c, 0x96, 0x2b, 0x57, 0x13, 0xc3, - 0x51, 0x0b, 0x71, 0x14, 0x15, 0xf3, 0x31, 0xf6, 0xfa, 0x9e, 0xf2, 0xbf, 0x23, 0x5f, 0x2f, 0xfe, -]; - static INIT: Once = Once::new(); fn init() { INIT.call_once(|| { @@ -36,7 +23,7 @@ fn js_interop_js_first() { init(); let work_dir = prepare_test_set(TEST_SET_JS_FIRST); js_step_1_create_hypercore(TEST_SET_JS_FIRST); - let hash = common::create_hypercore_hash(work_dir).expect("Could not hash directory"); + let hash = create_hypercore_hash(work_dir).expect("Could not hash directory"); assert_eq!(get_step_1_hash(), hash) } @@ -46,17 +33,14 @@ async fn js_interop_rs_first() { init(); let work_dir = prepare_test_set(TEST_SET_RS_FIRST); step_1_create_hypercore(&work_dir).await; - let _hash = common::create_hypercore_hash(work_dir).expect("Could not hash directory"); + let _hash = create_hypercore_hash(work_dir).expect("Could not hash directory"); // TODO: Make this match, only data does right now // assert_eq!(get_step_1_hash(), hash) } async fn step_1_create_hypercore(work_dir: &str) { let path = Path::new(work_dir).to_owned(); - let storage = Storage::new_disk(&path, false).await.unwrap(); - - let public_key = PublicKey::from_bytes(&TEST_PUBLIC_KEY_BYTES).unwrap(); - let secret_key = SecretKey::from_bytes(&TEST_SECRET_KEY_BYTES).unwrap(); + let storage = get_test_key_pair(); // let builder = FeedBuilder::new(public_key, storage); // let mut feed = builder.secret_key(secret_key).build().await.unwrap(); From 85bc33239af640dbfc31e537504b492cb06e8030 Mon Sep 17 00:00:00 2001 From: Timo Tiuraniemi Date: Wed, 24 Aug 2022 14:54:32 +0300 Subject: [PATCH 019/157] Move to storage_v10 to avoid excessive feature flagging --- src/common/mod.rs | 3 + src/{storage => common}/node.rs | 0 src/{hypercore.rs => core.rs} | 2 +- src/crypto/hash.rs | 2 +- src/crypto/merkle.rs | 2 +- src/feed.rs | 3 +- src/feed_builder.rs | 2 +- src/lib.rs | 18 +++-- src/prelude.rs | 9 ++- src/storage/mod.rs | 66 +---------------- src/storage_v10/mod.rs | 122 ++++++++++++++++++++++++++++++++ tests/common/mod.rs | 49 ++++++++----- tests/{hypercore.rs => core.rs} | 0 tests/js/interop.js | 16 +++-- tests/js/mod.rs | 2 +- tests/js_interop.rs | 55 ++++++++++---- 16 files changed, 239 insertions(+), 112 deletions(-) create mode 100644 src/common/mod.rs rename src/{storage => common}/node.rs (100%) rename src/{hypercore.rs => core.rs} (96%) create mode 100644 src/storage_v10/mod.rs rename tests/{hypercore.rs => core.rs} (100%) diff --git a/src/common/mod.rs b/src/common/mod.rs new file mode 100644 index 00000000..22acd666 --- /dev/null +++ b/src/common/mod.rs @@ -0,0 +1,3 @@ +mod node; + +pub use self::node::Node; diff --git a/src/storage/node.rs b/src/common/node.rs similarity index 100% rename from src/storage/node.rs rename to src/common/node.rs diff --git a/src/hypercore.rs b/src/core.rs similarity index 96% rename from src/hypercore.rs rename to src/core.rs index b04f0c5d..5fa12786 100644 --- a/src/hypercore.rs +++ b/src/core.rs @@ -1,6 +1,6 @@ //! Hypercore's main abstraction. Exposes an append-only, secure log structure. -pub use crate::storage::{PartialKeypair, Storage}; +pub use crate::storage_v10::{PartialKeypair, Storage}; use crate::{ crypto::{generate_keypair, PublicKey, SecretKey}, diff --git a/src/crypto/hash.rs b/src/crypto/hash.rs index dc8ba2a8..eb63127f 100644 --- a/src/crypto/hash.rs +++ b/src/crypto/hash.rs @@ -1,6 +1,6 @@ pub use blake2_rfc::blake2b::Blake2bResult; -use crate::storage::Node; +use crate::common::Node; use blake2_rfc::blake2b::Blake2b; use byteorder::{BigEndian, WriteBytesExt}; use ed25519_dalek::PublicKey; diff --git a/src/crypto/merkle.rs b/src/crypto/merkle.rs index a55a7a6a..bbd1cb22 100644 --- a/src/crypto/merkle.rs +++ b/src/crypto/merkle.rs @@ -1,5 +1,5 @@ +use crate::common::Node; use crate::crypto::Hash; -use crate::storage::Node; use merkle_tree_stream::{HashMethods, MerkleTreeStream, NodeKind, PartialNode}; use std::sync::Arc; diff --git a/src/feed.rs b/src/feed.rs index afa14876..7b6f9e61 100644 --- a/src/feed.rs +++ b/src/feed.rs @@ -1,8 +1,9 @@ //! Hypercore's main abstraction. Exposes an append-only, secure log structure. +pub use crate::common::Node; use crate::feed_builder::FeedBuilder; use crate::replicate::{Message, Peer}; -pub use crate::storage::{Node, NodeTrait, Storage, Store}; +pub use crate::storage::{NodeTrait, Storage, Store}; use crate::audit::Audit; use crate::bitfield::Bitfield; diff --git a/src/feed_builder.rs b/src/feed_builder.rs index 60744e43..05d110fb 100644 --- a/src/feed_builder.rs +++ b/src/feed_builder.rs @@ -52,7 +52,7 @@ where Bitfield::new() }; - use crate::storage::Node; + use crate::common::Node; let mut tree = TreeIndex::new(tree); let mut roots = vec![]; diff --git a/src/lib.rs b/src/lib.rs index 4c7db915..5544ba2d 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -44,6 +44,9 @@ pub mod compact_encoding; pub mod prelude; mod audit; +mod common; +#[cfg(feature = "v10")] +mod core; mod crypto; mod event; #[cfg(not(feature = "v10"))] @@ -51,13 +54,17 @@ mod feed; #[cfg(not(feature = "v10"))] mod feed_builder; #[cfg(feature = "v10")] -mod hypercore; -#[cfg(feature = "v10")] mod oplog; mod proof; mod replicate; +#[cfg(not(feature = "v10"))] mod storage; +#[cfg(feature = "v10")] +mod storage_v10; +pub use crate::common::Node; +#[cfg(feature = "v10")] +pub use crate::core::Hypercore; pub use crate::crypto::{generate_keypair, sign, verify, Signature}; pub use crate::event::Event; #[cfg(not(feature = "v10"))] @@ -66,10 +73,11 @@ pub use crate::feed::Feed; pub use crate::feed_builder::FeedBuilder; pub use crate::proof::Proof; pub use crate::replicate::Peer; -pub use crate::storage::{Node, NodeTrait, PartialKeypair, Storage, Store}; -pub use ed25519_dalek::{PublicKey, SecretKey}; +#[cfg(not(feature = "v10"))] +pub use crate::storage::{NodeTrait, PartialKeypair, Storage, Store}; #[cfg(feature = "v10")] -pub use hypercore::Hypercore; +pub use crate::storage_v10::{PartialKeypair, Storage, Store}; +pub use ed25519_dalek::{PublicKey, SecretKey}; use std::path::Path; diff --git a/src/prelude.rs b/src/prelude.rs index 2519a807..618ad483 100644 --- a/src/prelude.rs +++ b/src/prelude.rs @@ -8,4 +8,11 @@ #[cfg(not(feature = "v10"))] pub use crate::feed::Feed; // pub use feed_builder::FeedBuilder; -pub use crate::storage::{Node, NodeTrait, Storage, Store}; +#[cfg(not(feature = "v10"))] +pub use crate::common::Node; +#[cfg(feature = "v10")] +pub use crate::core::Hypercore; +#[cfg(not(feature = "v10"))] +pub use crate::storage::{NodeTrait, Storage, Store}; +#[cfg(feature = "v10")] +pub use crate::storage_v10::{PartialKeypair, Storage, Store}; diff --git a/src/storage/mod.rs b/src/storage/mod.rs index 92f9b88c..ff869518 100644 --- a/src/storage/mod.rs +++ b/src/storage/mod.rs @@ -1,12 +1,11 @@ //! Save data to a desired storage backend. -mod node; mod persist; -pub use self::node::Node; -pub use self::persist::Persist; pub use merkle_tree_stream::Node as NodeTrait; +pub use self::persist::Persist; +use crate::common::Node; use anyhow::{anyhow, ensure, Result}; use ed25519_dalek::{PublicKey, SecretKey, Signature, PUBLIC_KEY_LENGTH, SECRET_KEY_LENGTH}; use flat_tree as flat; @@ -33,22 +32,6 @@ pub struct PartialKeypair { pub secret: Option, } -impl Clone for PartialKeypair { - fn clone(&self) -> Self { - let secret: Option = match &self.secret { - Some(secret) => { - let bytes = secret.to_bytes(); - Some(SecretKey::from_bytes(&bytes).unwrap()) - } - None => None, - }; - PartialKeypair { - public: self.public.clone(), - secret, - } - } -} - /// The types of stores that can be created. #[derive(Debug)] pub enum Store { @@ -59,14 +42,9 @@ pub enum Store { /// Bitfield Bitfield, /// Signatures - #[cfg(not(feature = "v10"))] Signatures, /// Keypair - #[cfg(not(feature = "v10"))] Keypair, - /// Oplog - #[cfg(feature = "v10")] - Oplog, } /// Save data to a desired storage backend. @@ -78,12 +56,8 @@ where tree: T, data: T, bitfield: T, - #[cfg(not(feature = "v10"))] signatures: T, - #[cfg(not(feature = "v10"))] keypair: T, - #[cfg(feature = "v10")] - oplog: T, } impl Storage @@ -101,32 +75,17 @@ where let tree = create(Store::Tree).await?; let data = create(Store::Data).await?; let bitfield = create(Store::Bitfield).await?; - #[cfg(not(feature = "v10"))] let signatures = create(Store::Signatures).await?; - #[cfg(not(feature = "v10"))] let keypair = create(Store::Keypair).await?; - #[cfg(feature = "v10")] - let oplog = create(Store::Oplog).await?; - let mut instance = Self { tree, data, bitfield, - #[cfg(not(feature = "v10"))] signatures, - #[cfg(not(feature = "v10"))] keypair, - #[cfg(feature = "v10")] - oplog, }; - #[cfg(feature = "v10")] - if overwrite || instance.bitfield.len().await.unwrap_or(0) == 0 { - // Init the oplog as an empty file - instance.oplog.write(0, &[]).await.map_err(|e| anyhow!(e))?; - } - if overwrite || instance.bitfield.len().await.unwrap_or(0) == 0 { let header = create_bitfield(); instance @@ -136,7 +95,6 @@ where .map_err(|e| anyhow!(e))?; } - #[cfg(not(feature = "v10"))] if overwrite || instance.signatures.len().await.unwrap_or(0) == 0 { let header = create_signatures(); instance @@ -204,7 +162,6 @@ where } /// Search the signature stores for a `Signature`, starting at `index`. - #[cfg(not(feature = "v10"))] pub fn next_signature( &mut self, index: u64, @@ -228,7 +185,6 @@ where /// Get a `Signature` from the store. #[inline] - #[cfg(not(feature = "v10"))] pub async fn get_signature(&mut self, index: u64) -> Result { let bytes = self .signatures @@ -243,7 +199,6 @@ where /// TODO: Ensure the signature size is correct. /// NOTE: Should we create a `Signature` entry type? #[inline] - #[cfg(not(feature = "v10"))] pub async fn put_signature( &mut self, index: u64, @@ -370,16 +325,7 @@ where } } - /// Read the full oplog bytes. - #[cfg(feature = "v10")] - pub async fn read_oplog(&mut self) -> Result> { - let len = self.oplog.len().await.map_err(|e| anyhow!(e))?; - let buf = self.oplog.read(0, len).await.map_err(|e| anyhow!(e))?; - Ok(buf.into_boxed_slice()) - } - /// Read a public key from storage - #[cfg(not(feature = "v10"))] pub async fn read_public_key(&mut self) -> Result { let buf = self .keypair @@ -391,7 +337,6 @@ where } /// Read a secret key from storage - #[cfg(not(feature = "v10"))] pub async fn read_secret_key(&mut self) -> Result { let buf = self .keypair @@ -403,14 +348,12 @@ where } /// Write a public key to the storage - #[cfg(not(feature = "v10"))] pub async fn write_public_key(&mut self, public_key: &PublicKey) -> Result<()> { let buf: [u8; PUBLIC_KEY_LENGTH] = public_key.to_bytes(); self.keypair.write(0, &buf).await.map_err(|e| anyhow!(e)) } /// Write a secret key to the storage - #[cfg(not(feature = "v10"))] pub async fn write_secret_key(&mut self, secret_key: &SecretKey) -> Result<()> { let buf: [u8; SECRET_KEY_LENGTH] = secret_key.to_bytes(); self.keypair @@ -420,7 +363,6 @@ where } /// Tries to read a partial keypair (ie: with an optional secret_key) from the storage - #[cfg(not(feature = "v10"))] pub async fn read_partial_keypair(&mut self) -> Option { match self.read_public_key().await { Ok(public) => match self.read_secret_key().await { @@ -455,12 +397,8 @@ impl Storage { Store::Tree => "tree", Store::Data => "data", Store::Bitfield => "bitfield", - #[cfg(not(feature = "v10"))] Store::Signatures => "signatures", - #[cfg(not(feature = "v10"))] Store::Keypair => "key", - #[cfg(feature = "v10")] - Store::Oplog => "oplog", }; RandomAccessDisk::open(dir.as_path().join(name)).boxed() }; diff --git a/src/storage_v10/mod.rs b/src/storage_v10/mod.rs new file mode 100644 index 00000000..abbf8eb0 --- /dev/null +++ b/src/storage_v10/mod.rs @@ -0,0 +1,122 @@ +//! Save data to a desired storage backend. + +use anyhow::{anyhow, Result}; +use ed25519_dalek::{PublicKey, SecretKey}; +use futures::future::FutureExt; +#[cfg(not(target_arch = "wasm32"))] +use random_access_disk::RandomAccessDisk; +use random_access_memory::RandomAccessMemory; +use random_access_storage::RandomAccess; +use std::fmt::Debug; +use std::path::PathBuf; + +/// Key pair where for read-only hypercores the secret key can also be missing. +#[derive(Debug)] +pub struct PartialKeypair { + /// Public key + pub public: PublicKey, + /// Secret key. If None, the hypercore is read-only. + pub secret: Option, +} + +impl Clone for PartialKeypair { + fn clone(&self) -> Self { + let secret: Option = match &self.secret { + Some(secret) => { + let bytes = secret.to_bytes(); + Some(SecretKey::from_bytes(&bytes).unwrap()) + } + None => None, + }; + PartialKeypair { + public: self.public.clone(), + secret, + } + } +} + +/// The types of stores that can be created. +#[derive(Debug)] +pub enum Store { + /// Tree + Tree, + /// Data + Data, + /// Bitfield + Bitfield, + /// Oplog + Oplog, +} + +/// Save data to a desired storage backend. +#[derive(Debug)] +pub struct Storage +where + T: RandomAccess + Debug, +{ + tree: T, + data: T, + bitfield: T, + oplog: T, +} + +impl Storage +where + T: RandomAccess> + Debug + Send, +{ + /// Create a new instance. Takes a callback to create new storage instances and overwrite flag. + pub async fn open(create: Cb, overwrite: bool) -> Result + where + Cb: Fn(Store) -> std::pin::Pin> + Send>>, + { + if overwrite { + unimplemented!("Clearing storage not implemented"); + } + let tree = create(Store::Tree).await?; + let data = create(Store::Data).await?; + let bitfield = create(Store::Bitfield).await?; + let oplog = create(Store::Oplog).await?; + + let instance = Self { + tree, + data, + bitfield, + oplog, + }; + + Ok(instance) + } + + /// Read the full oplog bytes. + pub async fn read_oplog(&mut self) -> Result> { + let len = self.oplog.len().await.map_err(|e| anyhow!(e))?; + let buf = self.oplog.read(0, len).await.map_err(|e| anyhow!(e))?; + Ok(buf.into_boxed_slice()) + } +} + +impl Storage { + /// New storage backed by a `RandomAccessMemory` instance. + pub async fn new_memory() -> Result { + let create = |_| async { Ok(RandomAccessMemory::default()) }.boxed(); + // No reason to overwrite, as this is a new memory segment + Ok(Self::open(create, false).await?) + } +} + +#[cfg(not(target_arch = "wasm32"))] +impl Storage { + /// New storage backed by a `RandomAccessDisk` instance. + pub async fn new_disk(dir: &PathBuf, overwrite: bool) -> Result { + let storage = |storage: Store| { + let name = match storage { + Store::Tree => "tree", + Store::Data => "data", + Store::Bitfield => "bitfield", + Store::Oplog => "oplog", + }; + RandomAccessDisk::open(dir.as_path().join(name)).boxed() + }; + Ok(Self::open(storage, overwrite).await?) + } +} diff --git a/tests/common/mod.rs b/tests/common/mod.rs index e775ce4d..b82f5caa 100644 --- a/tests/common/mod.rs +++ b/tests/common/mod.rs @@ -1,4 +1,4 @@ -use hypercore::{self, PartialKeypair}; +use hypercore::PartialKeypair; use anyhow::Error; use ed25519_dalek::{PublicKey, SecretKey, PUBLIC_KEY_LENGTH, SECRET_KEY_LENGTH}; @@ -29,10 +29,10 @@ pub async fn create_feed(page_size: usize) -> Result, + pub data: Option, + pub oplog: Option, + pub tree: Option, } pub fn get_test_key_pair() -> PartialKeypair { @@ -41,24 +41,35 @@ pub fn get_test_key_pair() -> PartialKeypair { PartialKeypair { public, secret } } -pub fn create_hypercore_hash(dir: String) -> Result { - let bitfield = hash_file(format!("{}/bitfield", dir))?; - let data = hash_file(format!("{}/data", dir))?; - let oplog = hash_file(format!("{}/oplog", dir))?; - let tree = hash_file(format!("{}/tree", dir))?; - Ok(HypercoreHash { +pub fn create_hypercore_hash(dir: &str) -> HypercoreHash { + let bitfield = hash_file(format!("{}/bitfield", dir)); + let data = hash_file(format!("{}/data", dir)); + let oplog = hash_file(format!("{}/oplog", dir)); + let tree = hash_file(format!("{}/tree", dir)); + HypercoreHash { bitfield, data, oplog, tree, - }) + } } -pub fn hash_file(file: String) -> Result { - let mut hasher = Sha256::new(); - let mut file = std::fs::File::open(file)?; - - std::io::copy(&mut file, &mut hasher)?; - let hash_bytes = hasher.finalize(); - Ok(format!("{:X}", hash_bytes)) +pub fn hash_file(file: String) -> Option { + let path = std::path::Path::new(&file); + if !path.exists() { + None + } else { + let mut hasher = Sha256::new(); + let mut file = std::fs::File::open(path).unwrap(); + std::io::copy(&mut file, &mut hasher).unwrap(); + let hash_bytes = hasher.finalize(); + let hash = format!("{:X}", hash_bytes); + // Empty file has this hash, don't make a difference between missing and empty file. Rust + // is much easier and performant to write if the empty file is created. + if hash == "E3B0C44298FC1C149AFBF4C8996FB92427AE41E4649B934CA495991B7852B855".to_string() { + None + } else { + Some(format!("{:X}", hash_bytes)) + } + } } diff --git a/tests/hypercore.rs b/tests/core.rs similarity index 100% rename from tests/hypercore.rs rename to tests/core.rs diff --git a/tests/js/interop.js b/tests/js/interop.js index f1f19988..f62c54d1 100644 --- a/tests/js/interop.js +++ b/tests/js/interop.js @@ -23,17 +23,25 @@ if (process.argv.length !== 4) { } if (process.argv[2] === '1') { - step1(process.argv[3]).then(result => { + step1Create(process.argv[3]).then(result => { console.log("step1 ready", result); }); +} else if (process.argv[2] === '2'){ + step2AppendHelloWorld(process.argv[3]).then(result => { + console.log("step2 ready", result); + }); } else { console.error(`Invalid test step {}`, process.argv[2]); process.exit(2); } -async function step1(testSet) { - let core = new Hypercore(`work/${testSet}`, testKeyPair.publicKey, {keyPair: testKeyPair}); +async function step1Create(testSet) { + const core = new Hypercore(`work/${testSet}`, testKeyPair.publicKey, {keyPair: testKeyPair}); + await core.close(); +}; + +async function step2AppendHelloWorld(testSet) { + const core = new Hypercore(`work/${testSet}`, testKeyPair.publicKey, {keyPair: testKeyPair}); await core.append(['Hello', 'World']); await core.close(); - core = new Hypercore(`work/${testSet}`, testKeyPair.publicKey, {keyPair: testKeyPair}); }; diff --git a/tests/js/mod.rs b/tests/js/mod.rs index 8b312a19..790b0a04 100644 --- a/tests/js/mod.rs +++ b/tests/js/mod.rs @@ -35,7 +35,7 @@ pub fn prepare_test_set(test_set: &str) -> String { path } -pub fn js_step_1_create_hypercore(test_set: &str) { +pub fn js_step_1_create(test_set: &str) { let status = Command::new("npm") .current_dir("tests/js") .args(&["run", "step1", test_set]) diff --git a/tests/js_interop.rs b/tests/js_interop.rs index abd28f04..a630507f 100644 --- a/tests/js_interop.rs +++ b/tests/js_interop.rs @@ -2,8 +2,11 @@ mod common; mod js; use std::{path::Path, sync::Once}; +use anyhow::Result; use common::{create_hypercore_hash, get_test_key_pair}; -use js::{cleanup, install, js_step_1_create_hypercore, prepare_test_set}; +#[cfg(feature = "v10")] +use hypercore::{Hypercore, Storage}; +use js::{cleanup, install, js_step_1_create, prepare_test_set}; const TEST_SET_JS_FIRST: &str = "jsfirst"; const TEST_SET_RS_FIRST: &str = "rsfirst"; @@ -22,25 +25,32 @@ fn init() { fn js_interop_js_first() { init(); let work_dir = prepare_test_set(TEST_SET_JS_FIRST); - js_step_1_create_hypercore(TEST_SET_JS_FIRST); - let hash = create_hypercore_hash(work_dir).expect("Could not hash directory"); - assert_eq!(get_step_1_hash(), hash) + assert_eq!(get_step_0_hash(), create_hypercore_hash(&work_dir)); + js_step_1_create(TEST_SET_JS_FIRST); + assert_eq!(get_step_1_hash(), create_hypercore_hash(&work_dir)) } #[async_std::test] #[cfg_attr(not(feature = "js_interop_tests"), ignore)] -async fn js_interop_rs_first() { +#[cfg(feature = "v10")] +async fn js_interop_rs_first() -> Result<()> { init(); let work_dir = prepare_test_set(TEST_SET_RS_FIRST); - step_1_create_hypercore(&work_dir).await; - let _hash = create_hypercore_hash(work_dir).expect("Could not hash directory"); + assert_eq!(get_step_0_hash(), create_hypercore_hash(&work_dir)); + step_1_create(&work_dir).await?; + assert_eq!(get_step_1_hash(), create_hypercore_hash(&work_dir)); + let _hash = create_hypercore_hash(&work_dir); // TODO: Make this match, only data does right now // assert_eq!(get_step_1_hash(), hash) + Ok(()) } -async fn step_1_create_hypercore(work_dir: &str) { +#[cfg(feature = "v10")] +async fn step_1_create(work_dir: &str) -> Result<()> { let path = Path::new(work_dir).to_owned(); - let storage = get_test_key_pair(); + let key_pair = get_test_key_pair(); + let storage = Storage::new_disk(&path, false).await?; + let hypercore = Hypercore::new_with_key_pair(storage, key_pair); // let builder = FeedBuilder::new(public_key, storage); // let mut feed = builder.secret_key(secret_key).build().await.unwrap(); @@ -48,13 +58,32 @@ async fn step_1_create_hypercore(work_dir: &str) { // feed.append(b"Hello").await.unwrap(); // feed.append(b"World").await.unwrap(); // drop(feed); + Ok(()) +} + +fn get_step_0_hash() -> common::HypercoreHash { + common::HypercoreHash { + bitfield: None, + data: None, + oplog: None, + tree: None, + } } fn get_step_1_hash() -> common::HypercoreHash { common::HypercoreHash { - bitfield: "0E2E1FF956A39192CBB68D2212288FE75B32733AB0C442B9F0471E254A0382A2".into(), - data: "872E4E50CE9990D8B041330C47C9DDD11BEC6B503AE9386A99DA8584E9BB12C4".into(), - oplog: "E374F3CFEA62D333E3ADE22A24A0EA50E5AF09CF45E2DEDC0F56F5A214081156".into(), - tree: "8577B24ADC763F65D562CD11204F938229AD47F27915B0821C46A0470B80813A".into(), + bitfield: None, + data: None, + oplog: Some("C5C042D47C25465FA708BB0384C88485E1C1AF848FC5D9E6DE34FAF1E88E41A9".into()), + tree: None, + } +} + +fn get_step_2_hash() -> common::HypercoreHash { + common::HypercoreHash { + bitfield: Some("0E2E1FF956A39192CBB68D2212288FE75B32733AB0C442B9F0471E254A0382A2".into()), + data: Some("872E4E50CE9990D8B041330C47C9DDD11BEC6B503AE9386A99DA8584E9BB12C4".into()), + oplog: Some("E374F3CFEA62D333E3ADE22A24A0EA50E5AF09CF45E2DEDC0F56F5A214081156".into()), + tree: Some("8577B24ADC763F65D562CD11204F938229AD47F27915B0821C46A0470B80813A".into()), } } From 728e1b2bf79396f262bcc881682532af15fecd5d Mon Sep 17 00:00:00 2001 From: Timo Tiuraniemi Date: Tue, 30 Aug 2022 10:02:34 +0300 Subject: [PATCH 020/157] Milestone 1: opening a hypercore produces identical oplog --- Cargo.toml | 1 + src/common/buffer.rs | 8 + src/common/mod.rs | 2 + src/compact_encoding/mod.rs | 23 +-- src/core.rs | 16 +- src/oplog/mod.rs | 338 ++++++++++++++++++++++++++++++++---- src/storage_v10/mod.rs | 47 ++++- tests/compact_encoding.rs | 18 +- tests/js/interop.js | 2 +- tests/js_interop.rs | 5 +- 10 files changed, 389 insertions(+), 71 deletions(-) create mode 100644 src/common/buffer.rs diff --git a/Cargo.toml b/Cargo.toml index c2cea5d7..5f373dd1 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -37,6 +37,7 @@ tree-index = "0.6.0" bitfield-rle = "0.2.0" futures = "0.3.4" async-std = "1.5.0" +crc32fast = "1.3.2" [target.'cfg(not(target_arch = "wasm32"))'.dependencies] random-access-disk = "2.0.0" diff --git a/src/common/buffer.rs b/src/common/buffer.rs new file mode 100644 index 00000000..a3b7e829 --- /dev/null +++ b/src/common/buffer.rs @@ -0,0 +1,8 @@ +/// Represents a slice to a known buffer. Useful for indicating changes that should be made to random +/// access storages. Data value of None, indicates that the should be truncated to the start +/// position. +#[derive(Debug)] +pub struct BufferSlice { + pub(crate) index: u64, + pub(crate) data: Option>, +} diff --git a/src/common/mod.rs b/src/common/mod.rs index 22acd666..18644bf9 100644 --- a/src/common/mod.rs +++ b/src/common/mod.rs @@ -1,3 +1,5 @@ +mod buffer; mod node; +pub use self::buffer::BufferSlice; pub use self::node::Node; diff --git a/src/compact_encoding/mod.rs b/src/compact_encoding/mod.rs index 9731d8b2..87622dbb 100644 --- a/src/compact_encoding/mod.rs +++ b/src/compact_encoding/mod.rs @@ -19,27 +19,26 @@ pub struct State { impl State { /// Create emtpy state pub fn new() -> State { - State { start: 0, end: 0 } + State::new_with_start_and_end(0, 0) } /// Create a state with an already known size. /// With this, you can/must skip the preencode step. pub fn new_with_size(size: usize) -> (State, Box<[u8]>) { ( - State { - start: 0, - end: size, - }, + State::new_with_start_and_end(0, size), vec![0; size].into_boxed_slice(), ) } + /// Create a state with a start and end already known. + pub fn new_with_start_and_end(start: usize, end: usize) -> State { + State { start, end } + } + /// Create a state from existing buffer. pub fn from_buffer(buffer: &Box<[u8]>) -> State { - State { - start: 0, - end: buffer.len(), - } + State::new_with_start_and_end(0, buffer.len()) } /// After calling preencode(), this allocates the right size buffer to the heap. @@ -205,9 +204,11 @@ impl State { /// Decode a byte buffer pub fn decode_buffer(&mut self, buffer: &Box<[u8]>) -> Box<[u8]> { let len = self.decode_usize_var(buffer); - buffer[self.start..self.start + len] + let value = buffer[self.start..self.start + len] .to_vec() - .into_boxed_slice() + .into_boxed_slice(); + self.start += value.len(); + value } /// Preencode a string array diff --git a/src/core.rs b/src/core.rs index 5fa12786..96fd4d98 100644 --- a/src/core.rs +++ b/src/core.rs @@ -1,11 +1,8 @@ //! Hypercore's main abstraction. Exposes an append-only, secure log structure. -pub use crate::storage_v10::{PartialKeypair, Storage}; +pub use crate::storage_v10::{PartialKeypair, Storage, Store}; -use crate::{ - crypto::{generate_keypair, PublicKey, SecretKey}, - oplog::Oplog, -}; +use crate::{crypto::generate_keypair, oplog::Oplog}; use anyhow::Result; use random_access_storage::RandomAccess; use std::fmt::Debug; @@ -47,13 +44,16 @@ where mut storage: Storage, key_pair: PartialKeypair, ) -> Result> { - let oplog_bytes = storage.read_oplog().await?; - let oplog = Oplog::open(key_pair.clone(), oplog_bytes); + let oplog_bytes = storage.read_all(Store::Oplog).await?; + let oplog_open_outcome = Oplog::open(key_pair.clone(), oplog_bytes); + storage + .flush_slices(Store::Oplog, oplog_open_outcome.slices_to_flush) + .await?; Ok(Hypercore { key_pair, storage, - oplog, + oplog: oplog_open_outcome.oplog, }) } } diff --git a/src/oplog/mod.rs b/src/oplog/mod.rs index 0ed21cb9..257645dd 100644 --- a/src/oplog/mod.rs +++ b/src/oplog/mod.rs @@ -1,11 +1,17 @@ +use std::convert::TryInto; +use std::iter::Map; + +use crate::common::BufferSlice; use crate::compact_encoding::{CompactEncoding, State}; -use crate::crypto::{generate_keypair, PublicKey, SecretKey}; +use crate::crypto::{PublicKey, SecretKey}; use crate::PartialKeypair; -/// Oplog header +/// Oplog header. #[derive(Debug)] struct Header { types: HeaderTypes, + // TODO: This is a keyValueArray in JS + user_data: Vec, tree: HeaderTree, signer: PartialKeypair, hints: HeaderHints, @@ -16,12 +22,9 @@ impl Header { /// Creates a new Header from given key pair pub fn new(key_pair: PartialKeypair) -> Header { Header { - types: HeaderTypes { - tree: "blake2b".to_string(), - bitfield: "raw".to_string(), - signer: "ed25519".to_string(), - }, - tree: HeaderTree { fork: 0, length: 0 }, + types: HeaderTypes::new(), + user_data: vec![], + tree: HeaderTree::new(), signer: key_pair, hints: HeaderHints { reorgs: vec![] }, contiguous_length: 0, @@ -47,12 +50,21 @@ impl Header { } /// Oplog header types -#[derive(Debug)] +#[derive(Debug, PartialEq)] struct HeaderTypes { tree: String, bitfield: String, signer: String, } +impl HeaderTypes { + pub fn new() -> HeaderTypes { + HeaderTypes { + tree: "blake2b".to_string(), + bitfield: "raw".to_string(), + signer: "ed25519".to_string(), + } + } +} impl CompactEncoding for State { fn preencode(&mut self, value: &HeaderTypes) { @@ -68,9 +80,9 @@ impl CompactEncoding for State { } fn decode(&mut self, buffer: &Box<[u8]>) -> HeaderTypes { - let tree = self.decode(buffer); - let bitfield = self.decode(buffer); - let signer = self.decode(buffer); + let tree: String = self.decode(buffer); + let bitfield: String = self.decode(buffer); + let signer: String = self.decode(buffer); HeaderTypes { tree, bitfield, @@ -80,38 +92,63 @@ impl CompactEncoding for State { } /// Oplog header tree -#[derive(Debug)] +#[derive(Debug, PartialEq)] struct HeaderTree { fork: u64, length: u64, + root_hash: Box<[u8]>, + signature: Box<[u8]>, +} + +impl HeaderTree { + pub fn new() -> HeaderTree { + HeaderTree { + fork: 0, + length: 0, + root_hash: Box::new([]), + signature: Box::new([]), + } + } } impl CompactEncoding for State { fn preencode(&mut self, value: &HeaderTree) { self.preencode(&value.fork); self.preencode(&value.length); + self.preencode(&value.root_hash); + self.preencode(&value.signature); } fn encode(&mut self, value: &HeaderTree, buffer: &mut Box<[u8]>) { self.encode(&value.fork, buffer); self.encode(&value.length, buffer); + self.encode(&value.root_hash, buffer); + self.encode(&value.signature, buffer); } fn decode(&mut self, buffer: &Box<[u8]>) -> HeaderTree { - let fork = self.decode(buffer); - let length = self.decode(buffer); - HeaderTree { fork, length } + let fork: u64 = self.decode(buffer); + let length: u64 = self.decode(buffer); + let root_hash: Box<[u8]> = self.decode(buffer); + let signature: Box<[u8]> = self.decode(buffer); + HeaderTree { + fork, + length, + root_hash, + signature, + } } } +/// NB: In Javascript's sodium the secret key contains in itself also the public key, so to +/// maintain binary compatibility, we store the public key in the oplog now twice. impl CompactEncoding for State { fn preencode(&mut self, value: &PartialKeypair) { - let public_key_bytes: Box<[u8]> = value.public.as_bytes().to_vec().into_boxed_slice(); - self.preencode(&public_key_bytes); + self.end += 1 + 32; match &value.secret { - Some(secret_key) => { - let secret_key_bytes: Box<[u8]> = secret_key.as_bytes().to_vec().into_boxed_slice(); - self.preencode(&secret_key_bytes); + Some(_) => { + // Also add room for the public key + self.end += 1 + 64; } None => { self.end += 1; @@ -124,7 +161,10 @@ impl CompactEncoding for State { self.encode(&public_key_bytes, buffer); match &value.secret { Some(secret_key) => { - let secret_key_bytes: Box<[u8]> = secret_key.as_bytes().to_vec().into_boxed_slice(); + let mut secret_key_bytes: Vec = Vec::with_capacity(64); + secret_key_bytes.extend_from_slice(secret_key.as_bytes()); + secret_key_bytes.extend_from_slice(&public_key_bytes); + let secret_key_bytes: Box<[u8]> = secret_key_bytes.into_boxed_slice(); self.encode(&secret_key_bytes, buffer); } None => { @@ -140,7 +180,7 @@ impl CompactEncoding for State { let secret: Option = if secret_key_bytes.len() == 0 { None } else { - Some(SecretKey::from_bytes(&secret_key_bytes).unwrap()) + Some(SecretKey::from_bytes(&secret_key_bytes[0..32]).unwrap()) }; PartialKeypair { @@ -174,9 +214,9 @@ impl CompactEncoding for State { impl CompactEncoding
for State { fn preencode(&mut self, value: &Header) { - self.start += 1; // Version + self.end += 1; // Version self.preencode(&value.types); - // TODO self.preencode(&value.user_data); + self.preencode(&value.user_data); self.preencode(&value.tree); self.preencode(&value.signer); self.preencode(&value.hints); @@ -184,10 +224,10 @@ impl CompactEncoding
for State { } fn encode(&mut self, value: &Header, buffer: &mut Box<[u8]>) { - buffer[0] = 0; // Version + buffer[self.start] = 0; // Version self.start += 1; self.encode(&value.types, buffer); - // TODO self.encode(&value.user_data, buffer); + self.encode(&value.user_data, buffer); self.encode(&value.tree, buffer); self.encode(&value.signer, buffer); self.encode(&value.hints, buffer); @@ -195,12 +235,13 @@ impl CompactEncoding
for State { } fn decode(&mut self, buffer: &Box<[u8]>) -> Header { - let version: usize = self.decode(buffer); + let version: u8 = buffer[self.start]; + self.start += 1; if version != 0 { panic!("Unknown oplog version {}", version); } let types: HeaderTypes = self.decode(buffer); - // TODO: let user_data: HeaderUserData = self.decode(buffer); + let user_data: Vec = self.decode(buffer); let tree: HeaderTree = self.decode(buffer); let signer: PartialKeypair = self.decode(buffer); let hints: HeaderHints = self.decode(buffer); @@ -208,6 +249,7 @@ impl CompactEncoding
for State { Header { types, + user_data, tree, signer, hints, @@ -216,23 +258,249 @@ impl CompactEncoding
for State { } } -/// Oplog +/// Oplog. +/// +/// There are two memory areas for an `Header` in `RandomAccessStorage`: one is the current +/// and one is the older. Which one is used depends on the value stored in the eigth byte's +/// eight bit of the stored headers. #[derive(Debug)] pub struct Oplog { - #[allow(dead_code)] - header: Header, + header_bits: [bool; 2], + entries_len: u64, +} + +/// Oplog +#[derive(Debug)] +pub struct OplogOpenOutcome { + pub oplog: Oplog, + pub slices_to_flush: Vec, } +enum OplogSlot { + FirstHeader = 0, + SecondHeader = 4096, + Entries = 4096 * 2, +} + +// The first set of bits is [1, 0], see `get_next_header_oplog_slot_and_bit_value` for how +// they change. +const INITIAL_HEADER_BITS: [bool; 2] = [true, false]; + impl Oplog { /// Opens an new Oplog from given key pair and existing content as a byte buffer #[allow(dead_code)] - pub fn open(key_pair: PartialKeypair, existing: Box<[u8]>) -> Oplog { + pub fn open(key_pair: PartialKeypair, existing: Box<[u8]>) -> OplogOpenOutcome { if existing.len() == 0 { - Oplog { - header: Header::new(key_pair), + let oplog = Oplog { + header_bits: INITIAL_HEADER_BITS, + entries_len: 0, + }; + + // The first 8 bytes will be filled with `prepend_crc32_and_len_with_bits`. + let data_start_index: usize = 8; + let mut state = State::new_with_start_and_end(data_start_index, data_start_index); + + // Get the right slot and header bit + let (oplog_slot, header_bit) = + Oplog::get_next_header_oplog_slot_and_bit_value(&oplog.header_bits); + + // Preencode a new header + let header = Header::new(key_pair); + state.preencode(&header); + + // Create a buffer for the needed data + let mut buffer = state.create_buffer(); + + // Encode the header + state.encode(&header, &mut buffer); + + // Finally prepend the buffer's 8 first bytes with a CRC, len and right bits + Oplog::prepend_crc32_and_len_with_bits( + state.end - data_start_index, + header_bit, + false, + &mut state, + &mut buffer, + ); + + // JS has this: + // + // this.flushed = false + // this._headers[0] = 1 + // this._headers[1] = 0 + // + // const state = { start: 8, end: 8, buffer: null } + // const i = this._headers[0] === this._headers[1] ? 1 : 0 + // const headerBit = (this._headers[i] + 1) & 1 + // this.headerEncoding.preencode(state, header) + // state.buffer = b4a.allocUnsafe(state.end) + // this.headerEncoding.encode(state, header) + // const len = state.end - 8; + // const partialBit = 0; + // + // // add the uint header (frame length and flush info) + // state.start = state.start - len - 4 + // cenc.uint32.encode(state, (len << 2) | headerBit | partialBit) + // // crc32 the length + header-bit + content and prefix it + // state.start -= 8 + // cenc.uint32.encode(state, crc32(state.buffer.subarray(state.start + 4, state.start + 8 + len))) + // state.start += len + 4 + // + // this.storage.write(i === 0 ? 0 : 4096, buffer) + // this.storage.truncate(4096 * 2) + // this._headers[i] = headerBit + // this.byteLength = 0 + // this.length = 0 + // this.flushed = true + + // TODO: Things will need to be extracted out of this to be reusable elsewhere, but + // let's try to get the first save of oplog have identical bytes to JS first. + + // The oplog is always truncated to the minimum byte size, which is right after + // the all of the entries in the oplog finish. + let truncate_index = OplogSlot::Entries as u64 + oplog.entries_len; + OplogOpenOutcome { + oplog, + slices_to_flush: vec![ + BufferSlice { + index: oplog_slot as u64, + data: Some(buffer), + }, + BufferSlice { + index: truncate_index, + data: None, + }, + ], } } else { unimplemented!("Reading an exising oplog is not supported yet"); } } + + /// Prepends given `State` with 4 bytes of CRC followed by 4 bytes containing length of + /// following buffer, 1 bit indicating which header is relevant to the entry (or if used to + /// wrap the actual header, then the header bit relevant for saving) and 1 bit that tells if + /// the written batch is only partially finished. For this to work, the state given must have + /// 8 bytes in reserve in the beginning, so that state.start can be set back 8 bytes. + fn prepend_crc32_and_len_with_bits( + len: usize, + header_bit: bool, + partial_bit: bool, + state: &mut State, + buffer: &mut Box<[u8]>, + ) { + // The 4 bytes right before start of data is the length in 8+8+8+6=30 bits. The 31st bit is + // the partial bit and 32nd bit the header bit. + state.start = state.start - len - 4; + let len_u32: u32 = len.try_into().unwrap(); + let partial_bit: u32 = if partial_bit { 2 } else { 0 }; + let header_bit: u32 = if header_bit { 1 } else { 0 }; + let value: u32 = (len_u32 << 2) | header_bit | partial_bit; + state.encode_uint32(value, buffer); + + // Before that, is a 4 byte CRC32 that is a checksum of the above encoded 4 bytes and the + // content. + state.start = state.start - 8; + let checksum = crc32fast::hash(&buffer[state.start + 4..state.start + 8 + len]); + state.encode_uint32(checksum, buffer); + } + + /// Based on given header_bits, determines if saving the header should be done to the first + /// header slot or the second header slot and the bit that it should get. + fn get_next_header_oplog_slot_and_bit_value(header_bits: &[bool; 2]) -> (OplogSlot, bool) { + // Writing a header to the disk is most efficient when only one area is saved. + // This makes it a bit less obvious to find out which of the headers is older + // and which newer. The bits indicate the header slot index in this way: + // + // [true, false] => [false, false] => [false, true] => [true, true] => [true, false] ... + // 0 => 1 => 0 => 1 => 0 + if header_bits[0] != header_bits[1] { + // First slot + (OplogSlot::FirstHeader, !header_bits[0]) + } else { + // Second slot + (OplogSlot::SecondHeader, !header_bits[1]) + } + } +} + +#[cfg(test)] +mod tests { + use crate::{ + compact_encoding::{CompactEncoding, State}, + crypto::generate_keypair, + oplog::{Header, HeaderTree, HeaderTypes}, + PartialKeypair, + }; + + #[test] + fn encode_header_types() { + let mut enc_state = State::new_with_start_and_end(8, 8); + let header_types = HeaderTypes::new(); + enc_state.preencode(&header_types); + let mut buffer = enc_state.create_buffer(); + enc_state.encode(&header_types, &mut buffer); + let mut dec_state = State::from_buffer(&buffer); + dec_state.start = 8; + let header_types_ret: HeaderTypes = dec_state.decode(&buffer); + assert_eq!(header_types, header_types_ret); + } + + #[test] + fn encode_partial_key_pair() { + let mut enc_state = State::new(); + let key_pair = generate_keypair(); + let key_pair = PartialKeypair { + public: key_pair.public, + secret: Some(key_pair.secret), + }; + enc_state.preencode(&key_pair); + let mut buffer = enc_state.create_buffer(); + // Pub key: 1 byte for length, 32 bytes for content + // Sec key: 1 byte for length, 64 bytes for data + let expected_len = 1 + 32 + 1 + 64; + assert_eq!(buffer.len(), expected_len); + assert_eq!(enc_state.end, expected_len); + assert_eq!(enc_state.start, 0); + enc_state.encode(&key_pair, &mut buffer); + let mut dec_state = State::from_buffer(&buffer); + let key_pair_ret: PartialKeypair = dec_state.decode(&buffer); + assert_eq!(key_pair.public, key_pair_ret.public); + assert_eq!( + key_pair.secret.unwrap().as_bytes(), + key_pair_ret.secret.unwrap().as_bytes() + ); + } + + #[test] + fn encode_tree() { + let mut enc_state = State::new(); + let tree = HeaderTree::new(); + enc_state.preencode(&tree); + let mut buffer = enc_state.create_buffer(); + enc_state.encode(&tree, &mut buffer); + let mut dec_state = State::from_buffer(&buffer); + let tree_ret: HeaderTree = dec_state.decode(&buffer); + assert_eq!(tree, tree_ret); + } + + #[test] + fn encode_header() { + let mut enc_state = State::new(); + let key_pair = generate_keypair(); + let key_pair = PartialKeypair { + public: key_pair.public, + secret: Some(key_pair.secret), + }; + let header = Header::new(key_pair); + enc_state.preencode(&header); + let mut buffer = enc_state.create_buffer(); + enc_state.encode(&header, &mut buffer); + let mut dec_state = State::from_buffer(&buffer); + let header_ret: Header = dec_state.decode(&buffer); + assert_eq!(header.signer.public, header_ret.signer.public); + assert_eq!(header.tree.fork, header_ret.tree.fork); + assert_eq!(header.tree.length, header_ret.tree.length); + assert_eq!(header.types, header_ret.types); + } } diff --git a/src/storage_v10/mod.rs b/src/storage_v10/mod.rs index abbf8eb0..664c6f85 100644 --- a/src/storage_v10/mod.rs +++ b/src/storage_v10/mod.rs @@ -10,6 +10,8 @@ use random_access_storage::RandomAccess; use std::fmt::Debug; use std::path::PathBuf; +use crate::common::BufferSlice; + /// Key pair where for read-only hypercores the secret key can also be missing. #[derive(Debug)] pub struct PartialKeypair { @@ -87,12 +89,45 @@ where Ok(instance) } - /// Read the full oplog bytes. - pub async fn read_oplog(&mut self) -> Result> { - let len = self.oplog.len().await.map_err(|e| anyhow!(e))?; - let buf = self.oplog.read(0, len).await.map_err(|e| anyhow!(e))?; + /// Read fully a store. + pub async fn read_all(&mut self, store: Store) -> Result> { + let storage = self.get_random_access(store); + let len = storage.len().await.map_err(|e| anyhow!(e))?; + let buf = storage.read(0, len).await.map_err(|e| anyhow!(e))?; Ok(buf.into_boxed_slice()) } + + /// Flushes slices. Either writes directly to a random access storage or truncates the storage + /// to the length of given index. + pub async fn flush_slices(&mut self, store: Store, slices: Vec) -> Result<()> { + let storage = self.get_random_access(store); + for slice in slices { + match slice.data { + Some(data) => { + storage + .write(slice.index, &data.to_vec()) + .await + .map_err(|e| anyhow!(e))?; + } + None => { + storage + .truncate(slice.index) + .await + .map_err(|e| anyhow!(e))?; + } + } + } + Ok(()) + } + + fn get_random_access(&mut self, store: Store) -> &mut T { + match store { + Store::Tree => &mut self.tree, + Store::Data => &mut self.data, + Store::Bitfield => &mut self.bitfield, + Store::Oplog => &mut self.oplog, + } + } } impl Storage { @@ -108,8 +143,8 @@ impl Storage { impl Storage { /// New storage backed by a `RandomAccessDisk` instance. pub async fn new_disk(dir: &PathBuf, overwrite: bool) -> Result { - let storage = |storage: Store| { - let name = match storage { + let storage = |store: Store| { + let name = match store { Store::Tree => "tree", Store::Data => "data", Store::Bitfield => "bitfield", diff --git a/tests/compact_encoding.rs b/tests/compact_encoding.rs index f7870ee1..0d6042da 100644 --- a/tests/compact_encoding.rs +++ b/tests/compact_encoding.rs @@ -110,16 +110,22 @@ fn cenc_u64_as_u32() { #[test] fn cenc_buffer() { - let buf_value = vec![0xFF, 0x00].into_boxed_slice(); + let buf_value_1 = vec![0xFF, 0x00].into_boxed_slice(); + let buf_value_2 = vec![0xEE, 0x11, 0x22].into_boxed_slice(); let mut enc_state = State::new(); - enc_state.preencode(&buf_value); + enc_state.preencode(&buf_value_1); + enc_state.preencode(&buf_value_2); let mut buffer = enc_state.create_buffer(); // 1 byte for length, 2 bytes for data - assert_eq!(buffer.len(), 3); - enc_state.encode(&buf_value, &mut buffer); + // 1 byte for length, 3 bytes for data + assert_eq!(buffer.len(), 1 + 2 + 1 + 3); + enc_state.encode(&buf_value_1, &mut buffer); + enc_state.encode(&buf_value_2, &mut buffer); let mut dec_state = State::from_buffer(&buffer); - let buf_value_ret: Box<[u8]> = dec_state.decode(&buffer); - assert_eq!(buf_value, buf_value_ret); + let buf_value_1_ret: Box<[u8]> = dec_state.decode(&buffer); + let buf_value_2_ret: Box<[u8]> = dec_state.decode(&buffer); + assert_eq!(buf_value_1, buf_value_1_ret); + assert_eq!(buf_value_2, buf_value_2_ret); } #[test] diff --git a/tests/js/interop.js b/tests/js/interop.js index f62c54d1..3320c004 100644 --- a/tests/js/interop.js +++ b/tests/js/interop.js @@ -36,7 +36,7 @@ if (process.argv[2] === '1') { } async function step1Create(testSet) { - const core = new Hypercore(`work/${testSet}`, testKeyPair.publicKey, {keyPair: testKeyPair}); + let core = new Hypercore(`work/${testSet}`, testKeyPair.publicKey, {keyPair: testKeyPair}); await core.close(); }; diff --git a/tests/js_interop.rs b/tests/js_interop.rs index a630507f..bd4c9595 100644 --- a/tests/js_interop.rs +++ b/tests/js_interop.rs @@ -39,9 +39,6 @@ async fn js_interop_rs_first() -> Result<()> { assert_eq!(get_step_0_hash(), create_hypercore_hash(&work_dir)); step_1_create(&work_dir).await?; assert_eq!(get_step_1_hash(), create_hypercore_hash(&work_dir)); - let _hash = create_hypercore_hash(&work_dir); - // TODO: Make this match, only data does right now - // assert_eq!(get_step_1_hash(), hash) Ok(()) } @@ -50,7 +47,7 @@ async fn step_1_create(work_dir: &str) -> Result<()> { let path = Path::new(work_dir).to_owned(); let key_pair = get_test_key_pair(); let storage = Storage::new_disk(&path, false).await?; - let hypercore = Hypercore::new_with_key_pair(storage, key_pair); + let _hypercore = Hypercore::new_with_key_pair(storage, key_pair).await?; // let builder = FeedBuilder::new(public_key, storage); // let mut feed = builder.secret_key(secret_key).build().await.unwrap(); From eda0266746829fa384dc7fa5cc87572426945c88 Mon Sep 17 00:00:00 2001 From: Timo Tiuraniemi Date: Tue, 30 Aug 2022 11:16:57 +0300 Subject: [PATCH 021/157] Add test for appending Hello World in JS --- tests/js/mod.rs | 13 +++++++++++++ tests/js/package.json | 3 ++- tests/js_interop.rs | 4 +++- 3 files changed, 18 insertions(+), 2 deletions(-) diff --git a/tests/js/mod.rs b/tests/js/mod.rs index 790b0a04..189c60fa 100644 --- a/tests/js/mod.rs +++ b/tests/js/mod.rs @@ -47,3 +47,16 @@ pub fn js_step_1_create(test_set: &str) { "node step 1 did not run successfully" ); } + +pub fn js_step_2_append_hello_world(test_set: &str) { + let status = Command::new("npm") + .current_dir("tests/js") + .args(&["run", "step2", test_set]) + .status() + .expect("Unable to run npm run"); + assert_eq!( + Some(0), + status.code(), + "node step 2 did not run successfully" + ); +} diff --git a/tests/js/package.json b/tests/js/package.json index 1088532b..d90320c4 100644 --- a/tests/js/package.json +++ b/tests/js/package.json @@ -2,7 +2,8 @@ "name": "hypercore-js-interop-tests", "version": "0.0.1", "scripts": { - "step1": "node interop.js 1" + "step1": "node interop.js 1", + "step2": "node interop.js 2" }, "dependencies": { "hypercore": "file:../../../hypercore-js" diff --git a/tests/js_interop.rs b/tests/js_interop.rs index bd4c9595..c1cc1254 100644 --- a/tests/js_interop.rs +++ b/tests/js_interop.rs @@ -6,7 +6,7 @@ use anyhow::Result; use common::{create_hypercore_hash, get_test_key_pair}; #[cfg(feature = "v10")] use hypercore::{Hypercore, Storage}; -use js::{cleanup, install, js_step_1_create, prepare_test_set}; +use js::{cleanup, install, js_step_1_create, js_step_2_append_hello_world, prepare_test_set}; const TEST_SET_JS_FIRST: &str = "jsfirst"; const TEST_SET_RS_FIRST: &str = "rsfirst"; @@ -39,6 +39,8 @@ async fn js_interop_rs_first() -> Result<()> { assert_eq!(get_step_0_hash(), create_hypercore_hash(&work_dir)); step_1_create(&work_dir).await?; assert_eq!(get_step_1_hash(), create_hypercore_hash(&work_dir)); + js_step_2_append_hello_world(TEST_SET_RS_FIRST); + assert_eq!(get_step_2_hash(), create_hypercore_hash(&work_dir)); Ok(()) } From b178ff178da99ed8bcc0c668c41647bb66d79333 Mon Sep 17 00:00:00 2001 From: Timo Tiuraniemi Date: Tue, 30 Aug 2022 11:59:25 +0300 Subject: [PATCH 022/157] Simpler interfacing with javascript test code --- src/core.rs | 15 +++++++++++++++ tests/js/mod.rs | 20 ++++---------------- tests/js/package.json | 3 +-- tests/js_interop.rs | 32 +++++++++++++++++++------------- 4 files changed, 39 insertions(+), 31 deletions(-) diff --git a/src/core.rs b/src/core.rs index 96fd4d98..240e6cae 100644 --- a/src/core.rs +++ b/src/core.rs @@ -22,6 +22,13 @@ where // pub(crate) bitfield: Bitfield, } +/// Response from append, matches that of the Javascript result +#[derive(Debug)] +pub struct AppendOutcome { + pub length: u64, + pub byte_length: u64, +} + impl Hypercore where T: RandomAccess> + Debug + Send, @@ -56,4 +63,12 @@ where oplog: oplog_open_outcome.oplog, }) } + + /// Appends a given batch of bytes to the hypercore. + pub async fn append_batch(&mut self, batch: Vec<&[u8]>) -> Result { + Ok(AppendOutcome { + length: 0, + byte_length: 0, + }) + } } diff --git a/tests/js/mod.rs b/tests/js/mod.rs index 189c60fa..b80730ce 100644 --- a/tests/js/mod.rs +++ b/tests/js/mod.rs @@ -35,28 +35,16 @@ pub fn prepare_test_set(test_set: &str) -> String { path } -pub fn js_step_1_create(test_set: &str) { +pub fn js_run_step(step: u8, test_set: &str) { let status = Command::new("npm") .current_dir("tests/js") - .args(&["run", "step1", test_set]) + .args(&["run", "step", &step.to_string(), test_set]) .status() .expect("Unable to run npm run"); assert_eq!( Some(0), status.code(), - "node step 1 did not run successfully" - ); -} - -pub fn js_step_2_append_hello_world(test_set: &str) { - let status = Command::new("npm") - .current_dir("tests/js") - .args(&["run", "step2", test_set]) - .status() - .expect("Unable to run npm run"); - assert_eq!( - Some(0), - status.code(), - "node step 2 did not run successfully" + "node step {} did not run successfully", + step ); } diff --git a/tests/js/package.json b/tests/js/package.json index d90320c4..5396f8e6 100644 --- a/tests/js/package.json +++ b/tests/js/package.json @@ -2,8 +2,7 @@ "name": "hypercore-js-interop-tests", "version": "0.0.1", "scripts": { - "step1": "node interop.js 1", - "step2": "node interop.js 2" + "step": "node interop.js" }, "dependencies": { "hypercore": "file:../../../hypercore-js" diff --git a/tests/js_interop.rs b/tests/js_interop.rs index c1cc1254..c01dac6f 100644 --- a/tests/js_interop.rs +++ b/tests/js_interop.rs @@ -6,7 +6,7 @@ use anyhow::Result; use common::{create_hypercore_hash, get_test_key_pair}; #[cfg(feature = "v10")] use hypercore::{Hypercore, Storage}; -use js::{cleanup, install, js_step_1_create, js_step_2_append_hello_world, prepare_test_set}; +use js::{cleanup, install, js_run_step, prepare_test_set}; const TEST_SET_JS_FIRST: &str = "jsfirst"; const TEST_SET_RS_FIRST: &str = "rsfirst"; @@ -20,14 +20,17 @@ fn init() { }); } -#[test] +#[async_std::test] #[cfg_attr(not(feature = "js_interop_tests"), ignore)] -fn js_interop_js_first() { +async fn js_interop_js_first() -> Result<()> { init(); let work_dir = prepare_test_set(TEST_SET_JS_FIRST); assert_eq!(get_step_0_hash(), create_hypercore_hash(&work_dir)); - js_step_1_create(TEST_SET_JS_FIRST); - assert_eq!(get_step_1_hash(), create_hypercore_hash(&work_dir)) + js_run_step(1, TEST_SET_JS_FIRST); + assert_eq!(get_step_1_hash(), create_hypercore_hash(&work_dir)); + step_2_append_hello_world(&work_dir).await?; + assert_eq!(get_step_2_hash(), create_hypercore_hash(&work_dir)); + Ok(()) } #[async_std::test] @@ -39,7 +42,7 @@ async fn js_interop_rs_first() -> Result<()> { assert_eq!(get_step_0_hash(), create_hypercore_hash(&work_dir)); step_1_create(&work_dir).await?; assert_eq!(get_step_1_hash(), create_hypercore_hash(&work_dir)); - js_step_2_append_hello_world(TEST_SET_RS_FIRST); + js_run_step(2, TEST_SET_RS_FIRST); assert_eq!(get_step_2_hash(), create_hypercore_hash(&work_dir)); Ok(()) } @@ -49,14 +52,17 @@ async fn step_1_create(work_dir: &str) -> Result<()> { let path = Path::new(work_dir).to_owned(); let key_pair = get_test_key_pair(); let storage = Storage::new_disk(&path, false).await?; - let _hypercore = Hypercore::new_with_key_pair(storage, key_pair).await?; - - // let builder = FeedBuilder::new(public_key, storage); - // let mut feed = builder.secret_key(secret_key).build().await.unwrap(); + Hypercore::new_with_key_pair(storage, key_pair).await?; + Ok(()) +} - // feed.append(b"Hello").await.unwrap(); - // feed.append(b"World").await.unwrap(); - // drop(feed); +#[cfg(feature = "v10")] +async fn step_2_append_hello_world(work_dir: &str) -> Result<()> { + let path = Path::new(work_dir).to_owned(); + let key_pair = get_test_key_pair(); + let storage = Storage::new_disk(&path, false).await?; + let mut hypercore = Hypercore::new_with_key_pair(storage, key_pair).await?; + let _append_outcome = hypercore.append_batch(vec![b"Hello", b"World"]).await?; Ok(()) } From 611cc019ac3696c37e0944a47399838f54996208 Mon Sep 17 00:00:00 2001 From: Timo Tiuraniemi Date: Tue, 30 Aug 2022 16:19:21 +0300 Subject: [PATCH 023/157] Some work towards opening an existing oplog --- src/compact_encoding/mod.rs | 4 +- src/core.rs | 2 +- src/oplog/mod.rs | 220 +++++++++++++++++++++--------------- 3 files changed, 135 insertions(+), 91 deletions(-) diff --git a/src/compact_encoding/mod.rs b/src/compact_encoding/mod.rs index 87622dbb..071b1ec4 100644 --- a/src/compact_encoding/mod.rs +++ b/src/compact_encoding/mod.rs @@ -48,7 +48,7 @@ impl State { } /// Encode u32 to 4 LE bytes. - pub fn encode_uint32(&mut self, uint: u32, buffer: &mut Box<[u8]>) { + pub fn encode_u32(&mut self, uint: u32, buffer: &mut Box<[u8]>) { self.encode_uint32_bytes(&uint.to_le_bytes(), buffer); } @@ -109,7 +109,7 @@ impl State { } else { buffer[self.start] = U32_SIGNIFIER; self.start += 1; - self.encode_uint32(*value, buffer); + self.encode_u32(*value, buffer); } } diff --git a/src/core.rs b/src/core.rs index 240e6cae..9e2c2fc3 100644 --- a/src/core.rs +++ b/src/core.rs @@ -52,7 +52,7 @@ where key_pair: PartialKeypair, ) -> Result> { let oplog_bytes = storage.read_all(Store::Oplog).await?; - let oplog_open_outcome = Oplog::open(key_pair.clone(), oplog_bytes); + let oplog_open_outcome = Oplog::open(key_pair.clone(), oplog_bytes)?; storage .flush_slices(Store::Oplog, oplog_open_outcome.slices_to_flush) .await?; diff --git a/src/oplog/mod.rs b/src/oplog/mod.rs index 257645dd..1e265f5e 100644 --- a/src/oplog/mod.rs +++ b/src/oplog/mod.rs @@ -1,10 +1,9 @@ -use std::convert::TryInto; -use std::iter::Map; - use crate::common::BufferSlice; use crate::compact_encoding::{CompactEncoding, State}; use crate::crypto::{PublicKey, SecretKey}; use crate::PartialKeypair; +use anyhow::{anyhow, Result}; +use std::convert::{TryFrom, TryInto}; /// Oplog header. #[derive(Debug)] @@ -282,98 +281,108 @@ enum OplogSlot { Entries = 4096 * 2, } +#[derive(Debug)] +struct ValidateLeaderOutcome { + state: State, + header_bit: bool, + partial_bit: bool, +} + // The first set of bits is [1, 0], see `get_next_header_oplog_slot_and_bit_value` for how // they change. const INITIAL_HEADER_BITS: [bool; 2] = [true, false]; impl Oplog { - /// Opens an new Oplog from given key pair and existing content as a byte buffer + /// Opens an existing Oplog from existing byte buffer or creates a new one. #[allow(dead_code)] - pub fn open(key_pair: PartialKeypair, existing: Box<[u8]>) -> OplogOpenOutcome { - if existing.len() == 0 { + pub fn open(key_pair: PartialKeypair, existing: Box<[u8]>) -> Result { + let h1_outcome = Self::validate_leader(OplogSlot::FirstHeader as usize, &existing)?; + let h2_outcome = Self::validate_leader(OplogSlot::SecondHeader as usize, &existing)?; + if let Some(h1_outcome) = h1_outcome { + let header_bits: [bool; 2] = if let Some(h2_outcome) = h2_outcome { + [h1_outcome.header_bit, h2_outcome.header_bit] + } else { + // If the bits match, the next is saved to the second slot, see + // `get_next_header_oplog_slot_and_bit_value` for details. + [h1_outcome.header_bit, h1_outcome.header_bit] + }; let oplog = Oplog { - header_bits: INITIAL_HEADER_BITS, + header_bits, entries_len: 0, }; - - // The first 8 bytes will be filled with `prepend_crc32_and_len_with_bits`. - let data_start_index: usize = 8; - let mut state = State::new_with_start_and_end(data_start_index, data_start_index); - - // Get the right slot and header bit - let (oplog_slot, header_bit) = - Oplog::get_next_header_oplog_slot_and_bit_value(&oplog.header_bits); - - // Preencode a new header - let header = Header::new(key_pair); - state.preencode(&header); - - // Create a buffer for the needed data - let mut buffer = state.create_buffer(); - - // Encode the header - state.encode(&header, &mut buffer); - - // Finally prepend the buffer's 8 first bytes with a CRC, len and right bits - Oplog::prepend_crc32_and_len_with_bits( - state.end - data_start_index, - header_bit, - false, - &mut state, - &mut buffer, - ); - - // JS has this: - // - // this.flushed = false - // this._headers[0] = 1 - // this._headers[1] = 0 - // - // const state = { start: 8, end: 8, buffer: null } - // const i = this._headers[0] === this._headers[1] ? 1 : 0 - // const headerBit = (this._headers[i] + 1) & 1 - // this.headerEncoding.preencode(state, header) - // state.buffer = b4a.allocUnsafe(state.end) - // this.headerEncoding.encode(state, header) - // const len = state.end - 8; - // const partialBit = 0; - // - // // add the uint header (frame length and flush info) - // state.start = state.start - len - 4 - // cenc.uint32.encode(state, (len << 2) | headerBit | partialBit) - // // crc32 the length + header-bit + content and prefix it - // state.start -= 8 - // cenc.uint32.encode(state, crc32(state.buffer.subarray(state.start + 4, state.start + 8 + len))) - // state.start += len + 4 - // - // this.storage.write(i === 0 ? 0 : 4096, buffer) - // this.storage.truncate(4096 * 2) - // this._headers[i] = headerBit - // this.byteLength = 0 - // this.length = 0 - // this.flushed = true - - // TODO: Things will need to be extracted out of this to be reusable elsewhere, but - // let's try to get the first save of oplog have identical bytes to JS first. - - // The oplog is always truncated to the minimum byte size, which is right after - // the all of the entries in the oplog finish. - let truncate_index = OplogSlot::Entries as u64 + oplog.entries_len; - OplogOpenOutcome { + Ok(OplogOpenOutcome { oplog, - slices_to_flush: vec![ - BufferSlice { - index: oplog_slot as u64, - data: Some(buffer), - }, - BufferSlice { - index: truncate_index, - data: None, - }, - ], - } + slices_to_flush: vec![], + }) + } else if let Some(h2_outcome) = h2_outcome { + // This shouldn't happen because the first header is saved to the first slot + // but Javascript supports this so we should too. + + // When the bits don't match, the next is saved to the first slot, see + // `get_next_header_oplog_slot_and_bit_value` for details. + let header_bits: [bool; 2] = [!h2_outcome.header_bit, h2_outcome.header_bit]; + let oplog = Oplog { + header_bits, + entries_len: 0, + }; + Ok(OplogOpenOutcome { + oplog, + slices_to_flush: vec![], + }) } else { - unimplemented!("Reading an exising oplog is not supported yet"); + // There is nothing in the oplog, start from new. + Ok(Self::new(key_pair)) + } + } + + fn new(key_pair: PartialKeypair) -> OplogOpenOutcome { + let oplog = Oplog { + header_bits: INITIAL_HEADER_BITS, + entries_len: 0, + }; + + // The first 8 bytes will be filled with `prepend_leader`. + let data_start_index: usize = 8; + let mut state = State::new_with_start_and_end(data_start_index, data_start_index); + + // Get the right slot and header bit + let (oplog_slot, header_bit) = + Oplog::get_next_header_oplog_slot_and_bit_value(&oplog.header_bits); + + // Preencode a new header + let header = Header::new(key_pair); + state.preencode(&header); + + // Create a buffer for the needed data + let mut buffer = state.create_buffer(); + + // Encode the header + state.encode(&header, &mut buffer); + + // Finally prepend the buffer's 8 first bytes with a CRC, len and right bits + Self::prepend_leader( + state.end - data_start_index, + header_bit, + false, + &mut state, + &mut buffer, + ); + + // The oplog is always truncated to the minimum byte size, which is right after + // the all of the entries in the oplog finish. + let truncate_index = OplogSlot::Entries as u64 + oplog.entries_len; + OplogOpenOutcome { + oplog, + slices_to_flush: vec![ + BufferSlice { + index: oplog_slot as u64, + data: Some(buffer), + }, + BufferSlice { + index: truncate_index, + data: None, + }, + ], } } @@ -382,7 +391,7 @@ impl Oplog { /// wrap the actual header, then the header bit relevant for saving) and 1 bit that tells if /// the written batch is only partially finished. For this to work, the state given must have /// 8 bytes in reserve in the beginning, so that state.start can be set back 8 bytes. - fn prepend_crc32_and_len_with_bits( + fn prepend_leader( len: usize, header_bit: bool, partial_bit: bool, @@ -395,14 +404,49 @@ impl Oplog { let len_u32: u32 = len.try_into().unwrap(); let partial_bit: u32 = if partial_bit { 2 } else { 0 }; let header_bit: u32 = if header_bit { 1 } else { 0 }; - let value: u32 = (len_u32 << 2) | header_bit | partial_bit; - state.encode_uint32(value, buffer); + let combined: u32 = (len_u32 << 2) | header_bit | partial_bit; + state.encode_u32(combined, buffer); // Before that, is a 4 byte CRC32 that is a checksum of the above encoded 4 bytes and the // content. state.start = state.start - 8; let checksum = crc32fast::hash(&buffer[state.start + 4..state.start + 8 + len]); - state.encode_uint32(checksum, buffer); + state.encode_u32(checksum, buffer); + } + + /// Validates that leader at given index is valid, and returns header and partial bits and + /// `State` for the header/entry that the leader was for. + fn validate_leader(index: usize, buffer: &Box<[u8]>) -> Result> { + if buffer.len() < index + 8 { + return Ok(None); + } + let mut state = State::new_with_start_and_end(index, buffer.len()); + let stored_checksum: u32 = state.decode_u32(buffer); + let combined: u32 = state.decode_u32(buffer); + let len = usize::try_from(combined >> 2) + .expect("Attempted converting to a 32 bit usize on below 32 bit system"); + + // NB: In the Javascript version IIUC zero length is caught only with a mismatch + // of checksums, which is silently interpreted to only mean "no value". That doesn't sound good: + // better to throw an error on mismatch and let the caller at least log the problem. + if len == 0 || state.end - state.start < len { + return Ok(None); + } + let header_bit = combined & 1 == 1; + let partial_bit = combined & 2 == 2; + + state.start = index + 8; + state.end = state.start + len; + let calculated_checksum = crc32fast::hash(&buffer[index + 4..state.end]); + if calculated_checksum != stored_checksum { + return Err(anyhow!("Checksums do not match")); + }; + + Ok(Some(ValidateLeaderOutcome { + header_bit, + partial_bit, + state, + })) } /// Based on given header_bits, determines if saving the header should be done to the first From 2fd950b7125f411fed11ca94501923ed77cafc74 Mon Sep 17 00:00:00 2001 From: Timo Tiuraniemi Date: Wed, 31 Aug 2022 08:46:15 +0300 Subject: [PATCH 024/157] Oplog returns current header to core --- src/oplog/mod.rs | 73 ++++++++++++++++++++++++++++-------------------- 1 file changed, 43 insertions(+), 30 deletions(-) diff --git a/src/oplog/mod.rs b/src/oplog/mod.rs index 1e265f5e..8ea5610b 100644 --- a/src/oplog/mod.rs +++ b/src/oplog/mod.rs @@ -7,14 +7,14 @@ use std::convert::{TryFrom, TryInto}; /// Oplog header. #[derive(Debug)] -struct Header { - types: HeaderTypes, +pub struct Header { + pub(crate) types: HeaderTypes, // TODO: This is a keyValueArray in JS - user_data: Vec, - tree: HeaderTree, - signer: PartialKeypair, - hints: HeaderHints, - contiguous_length: u64, + pub(crate) user_data: Vec, + pub(crate) tree: HeaderTree, + pub(crate) signer: PartialKeypair, + pub(crate) hints: HeaderHints, + pub(crate) contiguous_length: u64, } impl Header { @@ -50,10 +50,10 @@ impl Header { /// Oplog header types #[derive(Debug, PartialEq)] -struct HeaderTypes { - tree: String, - bitfield: String, - signer: String, +pub struct HeaderTypes { + pub(crate) tree: String, + pub(crate) bitfield: String, + pub(crate) signer: String, } impl HeaderTypes { pub fn new() -> HeaderTypes { @@ -92,11 +92,11 @@ impl CompactEncoding for State { /// Oplog header tree #[derive(Debug, PartialEq)] -struct HeaderTree { - fork: u64, - length: u64, - root_hash: Box<[u8]>, - signature: Box<[u8]>, +pub struct HeaderTree { + pub(crate) fork: u64, + pub(crate) length: u64, + pub(crate) root_hash: Box<[u8]>, + pub(crate) signature: Box<[u8]>, } impl HeaderTree { @@ -191,8 +191,8 @@ impl CompactEncoding for State { /// Oplog header hints #[derive(Debug)] -struct HeaderHints { - reorgs: Vec, +pub struct HeaderHints { + pub(crate) reorgs: Vec, } impl CompactEncoding for State { @@ -272,6 +272,7 @@ pub struct Oplog { #[derive(Debug)] pub struct OplogOpenOutcome { pub oplog: Oplog, + pub header: Header, pub slices_to_flush: Vec, } @@ -296,30 +297,40 @@ impl Oplog { /// Opens an existing Oplog from existing byte buffer or creates a new one. #[allow(dead_code)] pub fn open(key_pair: PartialKeypair, existing: Box<[u8]>) -> Result { + // First read and validate both headers stored in the existing oplog let h1_outcome = Self::validate_leader(OplogSlot::FirstHeader as usize, &existing)?; let h2_outcome = Self::validate_leader(OplogSlot::SecondHeader as usize, &existing)?; - if let Some(h1_outcome) = h1_outcome { - let header_bits: [bool; 2] = if let Some(h2_outcome) = h2_outcome { - [h1_outcome.header_bit, h2_outcome.header_bit] - } else { - // If the bits match, the next is saved to the second slot, see - // `get_next_header_oplog_slot_and_bit_value` for details. - [h1_outcome.header_bit, h1_outcome.header_bit] - }; + + // Depending on what is stored, the state needs to be set accordingly. + // See `get_next_header_oplog_slot_and_bit_value` for details on header_bits. + if let Some(mut h1_outcome) = h1_outcome { + let (header, header_bits): (Header, [bool; 2]) = + if let Some(mut h2_outcome) = h2_outcome { + let header_bits = [h1_outcome.header_bit, h2_outcome.header_bit]; + let header: Header = if header_bits[0] == header_bits[1] { + h1_outcome.state.decode(&existing) + } else { + h2_outcome.state.decode(&existing) + }; + (header, header_bits) + } else { + ( + h1_outcome.state.decode(&existing), + [h1_outcome.header_bit, h1_outcome.header_bit], + ) + }; let oplog = Oplog { header_bits, entries_len: 0, }; Ok(OplogOpenOutcome { oplog, + header, slices_to_flush: vec![], }) - } else if let Some(h2_outcome) = h2_outcome { + } else if let Some(mut h2_outcome) = h2_outcome { // This shouldn't happen because the first header is saved to the first slot // but Javascript supports this so we should too. - - // When the bits don't match, the next is saved to the first slot, see - // `get_next_header_oplog_slot_and_bit_value` for details. let header_bits: [bool; 2] = [!h2_outcome.header_bit, h2_outcome.header_bit]; let oplog = Oplog { header_bits, @@ -327,6 +338,7 @@ impl Oplog { }; Ok(OplogOpenOutcome { oplog, + header: h2_outcome.state.decode(&existing), slices_to_flush: vec![], }) } else { @@ -373,6 +385,7 @@ impl Oplog { let truncate_index = OplogSlot::Entries as u64 + oplog.entries_len; OplogOpenOutcome { oplog, + header, slices_to_flush: vec![ BufferSlice { index: oplog_slot as u64, From af9a46733f7d49e32f1ad3b4ff40751eb69df127 Mon Sep 17 00:00:00 2001 From: Timo Tiuraniemi Date: Wed, 31 Aug 2022 13:56:56 +0300 Subject: [PATCH 025/157] Attempt at moving to the blake2 crate, but alas, it lacks features. --- Cargo.toml | 4 ++++ src/core.rs | 5 +++++ src/oplog/mod.rs | 2 +- 3 files changed, 10 insertions(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index 5f373dd1..0479a5f8 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -18,6 +18,10 @@ categories = [ edition = "2018" [dependencies] +# TODO: After this issue: +# https://github.com/RustCrypto/hashes/issues/360 +# that blocks public key => discovery key creation is fixed, switching to the blake2 crate +# would probably be wise. blake2-rfc = "0.2.18" byteorder = "1.3.4" ed25519-dalek = "1.0.1" diff --git a/src/core.rs b/src/core.rs index 9e2c2fc3..18d992ca 100644 --- a/src/core.rs +++ b/src/core.rs @@ -66,6 +66,11 @@ where /// Appends a given batch of bytes to the hypercore. pub async fn append_batch(&mut self, batch: Vec<&[u8]>) -> Result { + let secret_key = match &self.key_pair.secret { + Some(key) => key, + None => anyhow::bail!("No secret key, cannot append."), + }; + Ok(AppendOutcome { length: 0, byte_length: 0, diff --git a/src/oplog/mod.rs b/src/oplog/mod.rs index 8ea5610b..4d1ab57a 100644 --- a/src/oplog/mod.rs +++ b/src/oplog/mod.rs @@ -470,7 +470,7 @@ impl Oplog { // and which newer. The bits indicate the header slot index in this way: // // [true, false] => [false, false] => [false, true] => [true, true] => [true, false] ... - // 0 => 1 => 0 => 1 => 0 + // First => Second => First => Second => First if header_bits[0] != header_bits[1] { // First slot (OplogSlot::FirstHeader, !header_bits[0]) From 1c70fb27344155c8f1141208651c26b152ae5329 Mon Sep 17 00:00:00 2001 From: Timo Tiuraniemi Date: Wed, 31 Aug 2022 15:20:44 +0300 Subject: [PATCH 026/157] Add v10 LE implementations of hashing --- src/compact_encoding/mod.rs | 15 ++++-- src/crypto/hash.rs | 98 +++++++++++++++++++++++++++++++++++++ 2 files changed, 108 insertions(+), 5 deletions(-) diff --git a/src/compact_encoding/mod.rs b/src/compact_encoding/mod.rs index 071b1ec4..73ea4318 100644 --- a/src/compact_encoding/mod.rs +++ b/src/compact_encoding/mod.rs @@ -47,11 +47,6 @@ impl State { vec![0; self.end].into_boxed_slice() } - /// Encode u32 to 4 LE bytes. - pub fn encode_u32(&mut self, uint: u32, buffer: &mut Box<[u8]>) { - self.encode_uint32_bytes(&uint.to_le_bytes(), buffer); - } - /// Preencode a string slice pub fn preencode_str(&mut self, value: &str) { self.preencode_usize_var(&value.len()); @@ -113,6 +108,11 @@ impl State { } } + /// Encode u32 to 4 LE bytes. + pub fn encode_u32(&mut self, uint: u32, buffer: &mut Box<[u8]>) { + self.encode_uint32_bytes(&uint.to_le_bytes(), buffer); + } + /// Decode a variable length u32 pub fn decode_u32_var(&mut self, buffer: &Box<[u8]>) -> u32 { let first = buffer[self.start]; @@ -157,6 +157,11 @@ impl State { } } + /// Encode u64 to 8 LE bytes. + pub fn encode_u64(&mut self, uint: u64, buffer: &mut Box<[u8]>) { + self.encode_uint64_bytes(&uint.to_le_bytes(), buffer); + } + /// Decode a variable length u64 pub fn decode_u64_var(&mut self, buffer: &Box<[u8]>) -> u64 { let first = buffer[self.start]; diff --git a/src/crypto/hash.rs b/src/crypto/hash.rs index eb63127f..6b820911 100644 --- a/src/crypto/hash.rs +++ b/src/crypto/hash.rs @@ -1,6 +1,7 @@ pub use blake2_rfc::blake2b::Blake2bResult; use crate::common::Node; +use crate::compact_encoding::{CompactEncoding, State}; use blake2_rfc::blake2b::Blake2b; use byteorder::{BigEndian, WriteBytesExt}; use ed25519_dalek::PublicKey; @@ -89,6 +90,68 @@ impl Hash { pub fn as_bytes(&self) -> &[u8] { self.hash.as_bytes() } + + // NB: The following methods mirror Javascript naming in + // https://github.com/mafintosh/hypercore-crypto/blob/master/index.js + // for v10 that use LE bytes. + + /// Hash data + pub fn data(data: &[u8]) -> Self { + let (mut state, mut size) = State::new_with_size(8); + state.encode_u64(data.len() as u64, &mut size); + + let mut hasher = Blake2b::new(32); + hasher.update(&LEAF_TYPE); + hasher.update(&size); + hasher.update(data); + + Self { + hash: hasher.finalize(), + } + } + + /// Hash a parent + pub fn parent(left: &Node, right: &Node) -> Self { + let (node1, node2) = if left.index <= right.index { + (left, right) + } else { + (right, left) + }; + + let (mut state, mut size) = State::new_with_size(8); + state.encode_u64((node1.length + node2.length) as u64, &mut size); + + let mut hasher = Blake2b::new(32); + hasher.update(&PARENT_TYPE); + hasher.update(&size); + hasher.update(node1.hash()); + hasher.update(node2.hash()); + + Self { + hash: hasher.finalize(), + } + } + + /// Hash a tree + pub fn tree(roots: &[impl AsRef]) -> Self { + let mut hasher = Blake2b::new(32); + hasher.update(&ROOT_TYPE); + + for node in roots { + let node = node.as_ref(); + let (mut state, mut buffer) = State::new_with_size(16); + state.encode_u64(node.index() as u64, &mut buffer); + state.encode_u64(node.len() as u64, &mut buffer); + + hasher.update(node.hash()); + hasher.update(&buffer[..8]); + hasher.update(&buffer[8..]); + } + + Self { + hash: hasher.finalize(), + } + } } fn u64_as_be(n: u64) -> [u8; 8] { @@ -186,4 +249,39 @@ mod tests { Ok(()) } + + // The following uses test data from + // https://github.com/mafintosh/hypercore-crypto/blob/master/test.js + + #[test] + fn hash_leaf() { + let data = b"hello world"; + check_hash( + Hash::data(data), + "9f1b578fd57a4df015493d2886aec9600eef913c3bb009768c7f0fb875996308", + ); + } + + #[test] + fn hash_parent() { + let data = b"hello world"; + let len = data.len() as u64; + let node1 = Node::new(0, Hash::data(data).as_bytes().to_vec(), len); + let node2 = Node::new(1, Hash::data(data).as_bytes().to_vec(), len); + check_hash( + Hash::parent(&node1, &node2), + "3ad0c9b58b771d1b7707e1430f37c23a23dd46e0c7c3ab9c16f79d25f7c36804", + ); + } + + #[test] + fn hash_tree() { + let hash: [u8; 32] = [0; 32]; + let node1 = Node::new(3, hash.to_vec(), 11); + let node2 = Node::new(9, hash.to_vec(), 2); + check_hash( + Hash::tree(&[&node1, &node2]), + "0e576a56b478cddb6ffebab8c494532b6de009466b2e9f7af9143fc54b9eaa36", + ); + } } From 6e8690ddd71129b10c95ab7caaa4656477e3839a Mon Sep 17 00:00:00 2001 From: Timo Tiuraniemi Date: Thu, 1 Sep 2022 13:48:52 +0300 Subject: [PATCH 027/157] Some work on reading the Merkle tree --- src/common/buffer.rs | 7 +++++++ src/common/mod.rs | 2 +- src/core.rs | 13 ++++++++++--- src/lib.rs | 2 ++ src/oplog/mod.rs | 9 +++++---- src/storage_v10/mod.rs | 30 ++++++++++++++++++++++++++---- src/tree/merkle_tree.rs | 34 ++++++++++++++++++++++++++++++++++ src/tree/merkle_tree_batch.rs | 4 ++++ src/tree/mod.rs | 5 +++++ 9 files changed, 94 insertions(+), 12 deletions(-) create mode 100644 src/tree/merkle_tree.rs create mode 100644 src/tree/merkle_tree_batch.rs create mode 100644 src/tree/mod.rs diff --git a/src/common/buffer.rs b/src/common/buffer.rs index a3b7e829..5850b0a2 100644 --- a/src/common/buffer.rs +++ b/src/common/buffer.rs @@ -6,3 +6,10 @@ pub struct BufferSlice { pub(crate) index: u64, pub(crate) data: Option>, } + +/// Represents an instruction to read a known buffer. +#[derive(Debug)] +pub struct BufferSliceInstruction { + pub(crate) index: u64, + pub(crate) len: u64, +} diff --git a/src/common/mod.rs b/src/common/mod.rs index 18644bf9..923f1560 100644 --- a/src/common/mod.rs +++ b/src/common/mod.rs @@ -1,5 +1,5 @@ mod buffer; mod node; -pub use self::buffer::BufferSlice; +pub use self::buffer::{BufferSlice, BufferSliceInstruction}; pub use self::node::Node; diff --git a/src/core.rs b/src/core.rs index 18d992ca..a1f7fea8 100644 --- a/src/core.rs +++ b/src/core.rs @@ -2,7 +2,7 @@ pub use crate::storage_v10::{PartialKeypair, Storage, Store}; -use crate::{crypto::generate_keypair, oplog::Oplog}; +use crate::{crypto::generate_keypair, oplog::Oplog, tree::MerkleTree}; use anyhow::Result; use random_access_storage::RandomAccess; use std::fmt::Debug; @@ -16,8 +16,7 @@ where pub(crate) key_pair: PartialKeypair, pub(crate) storage: Storage, pub(crate) oplog: Oplog, - // /// Merkle tree instance. - // pub(crate) tree: Merkle, + pub(crate) tree: MerkleTree, // /// Bitfield to keep track of which data we own. // pub(crate) bitfield: Bitfield, } @@ -51,16 +50,24 @@ where mut storage: Storage, key_pair: PartialKeypair, ) -> Result> { + // Open/create oplog let oplog_bytes = storage.read_all(Store::Oplog).await?; let oplog_open_outcome = Oplog::open(key_pair.clone(), oplog_bytes)?; storage .flush_slices(Store::Oplog, oplog_open_outcome.slices_to_flush) .await?; + // Open/create tree + let header_tree_length = oplog_open_outcome.header.tree.length; + let slice_instructions = MerkleTree::get_slice_instructions_to_read(header_tree_length); + let slices = storage.read_slices(Store::Tree, slice_instructions).await?; + let tree = MerkleTree::open(header_tree_length, slices)?; + Ok(Hypercore { key_pair, storage, oplog: oplog_open_outcome.oplog, + tree, }) } diff --git a/src/lib.rs b/src/lib.rs index 5544ba2d..a5c15cbe 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -61,6 +61,8 @@ mod replicate; mod storage; #[cfg(feature = "v10")] mod storage_v10; +#[cfg(feature = "v10")] +mod tree; pub use crate::common::Node; #[cfg(feature = "v10")] diff --git a/src/oplog/mod.rs b/src/oplog/mod.rs index 4d1ab57a..38c7a8cf 100644 --- a/src/oplog/mod.rs +++ b/src/oplog/mod.rs @@ -273,7 +273,7 @@ pub struct Oplog { pub struct OplogOpenOutcome { pub oplog: Oplog, pub header: Header, - pub slices_to_flush: Vec, + pub slices_to_flush: Box<[BufferSlice]>, } enum OplogSlot { @@ -326,7 +326,7 @@ impl Oplog { Ok(OplogOpenOutcome { oplog, header, - slices_to_flush: vec![], + slices_to_flush: Box::new([]), }) } else if let Some(mut h2_outcome) = h2_outcome { // This shouldn't happen because the first header is saved to the first slot @@ -339,7 +339,7 @@ impl Oplog { Ok(OplogOpenOutcome { oplog, header: h2_outcome.state.decode(&existing), - slices_to_flush: vec![], + slices_to_flush: Box::new([]), }) } else { // There is nothing in the oplog, start from new. @@ -395,7 +395,8 @@ impl Oplog { index: truncate_index, data: None, }, - ], + ] + .into_boxed_slice(), } } diff --git a/src/storage_v10/mod.rs b/src/storage_v10/mod.rs index 664c6f85..a856dfd8 100644 --- a/src/storage_v10/mod.rs +++ b/src/storage_v10/mod.rs @@ -10,7 +10,7 @@ use random_access_storage::RandomAccess; use std::fmt::Debug; use std::path::PathBuf; -use crate::common::BufferSlice; +use crate::common::{BufferSlice, BufferSliceInstruction}; /// Key pair where for read-only hypercores the secret key can also be missing. #[derive(Debug)] @@ -97,12 +97,34 @@ where Ok(buf.into_boxed_slice()) } + /// Read slices from a store based on given instructions + pub async fn read_slices( + &mut self, + store: Store, + slice_instructions: Box<[BufferSliceInstruction]>, + ) -> Result> { + let storage = self.get_random_access(store); + + let mut slices: Vec = Vec::with_capacity(slice_instructions.len()); + for instruction in slice_instructions.iter() { + let buf = storage + .read(instruction.index, instruction.len) + .await + .map_err(|e| anyhow!(e))?; + slices.push(BufferSlice { + index: instruction.index, + data: Some(buf.into_boxed_slice()), + }); + } + Ok(slices.into_boxed_slice()) + } + /// Flushes slices. Either writes directly to a random access storage or truncates the storage /// to the length of given index. - pub async fn flush_slices(&mut self, store: Store, slices: Vec) -> Result<()> { + pub async fn flush_slices(&mut self, store: Store, slices: Box<[BufferSlice]>) -> Result<()> { let storage = self.get_random_access(store); - for slice in slices { - match slice.data { + for slice in slices.iter() { + match &slice.data { Some(data) => { storage .write(slice.index, &data.to_vec()) diff --git a/src/tree/merkle_tree.rs b/src/tree/merkle_tree.rs new file mode 100644 index 00000000..05f69b5c --- /dev/null +++ b/src/tree/merkle_tree.rs @@ -0,0 +1,34 @@ +use anyhow::Result; + +use crate::common::{BufferSlice, BufferSliceInstruction}; + +/// Merkle tree. +/// See https://github.com/hypercore-protocol/hypercore/blob/master/lib/merkle-tree.js +#[derive(Debug)] +pub struct MerkleTree {} + +impl MerkleTree { + /// Gets instructions to slices that should be read from storage + /// based on `HeaderTree` `length` field. Call this before + /// calling `open`. + pub fn get_slice_instructions_to_read( + header_tree_length: u64, + ) -> Box<[BufferSliceInstruction]> { + let mut roots = vec![]; + flat_tree::full_roots(header_tree_length * 2, &mut roots); + + roots + .iter() + .map(|&index| BufferSliceInstruction { index, len: 8 }) + .collect::>() + .into_boxed_slice() + } + + /// Opens MerkleTree, based on read byte slices. Call `get_slice_instructions_to_read` + /// before calling this to find out which slices to read. + pub fn open(header_tree_length: u64, slices: Box<[BufferSlice]>) -> Result { + // read_tree_bytes(0, 0).await?; + // read_tree_bytes(0, 0)?; + Ok(Self {}) + } +} diff --git a/src/tree/merkle_tree_batch.rs b/src/tree/merkle_tree_batch.rs new file mode 100644 index 00000000..ed9fb5ef --- /dev/null +++ b/src/tree/merkle_tree_batch.rs @@ -0,0 +1,4 @@ +/// Merkle tree batch. +/// See https://github.com/hypercore-protocol/hypercore/blob/master/lib/merkle-tree.js +#[derive(Debug)] +pub struct MerkleTreeBatch {} diff --git a/src/tree/mod.rs b/src/tree/mod.rs new file mode 100644 index 00000000..54eb79cb --- /dev/null +++ b/src/tree/mod.rs @@ -0,0 +1,5 @@ +mod merkle_tree; +mod merkle_tree_batch; + +pub use merkle_tree::MerkleTree; +pub use merkle_tree_batch::MerkleTreeBatch; From f5f6f730d1faed01948244bf2a8c0e2dcaa63277 Mon Sep 17 00:00:00 2001 From: Timo Tiuraniemi Date: Fri, 2 Sep 2022 10:15:36 +0300 Subject: [PATCH 028/157] Some more work on tree --- src/common/buffer.rs | 9 ++++++ src/common/node.rs | 6 ++++ src/compact_encoding/mod.rs | 34 ++++++++++----------- src/core.rs | 2 +- src/oplog/mod.rs | 10 +++---- src/tree/merkle_tree.rs | 60 ++++++++++++++++++++++++++++++------- 6 files changed, 87 insertions(+), 34 deletions(-) diff --git a/src/common/buffer.rs b/src/common/buffer.rs index 5850b0a2..07e3c64a 100644 --- a/src/common/buffer.rs +++ b/src/common/buffer.rs @@ -7,6 +7,15 @@ pub struct BufferSlice { pub(crate) data: Option>, } +impl BufferSlice { + pub fn get_data_mut(&self) -> Box<[u8]> { + let data = self.data.as_ref().unwrap(); + let mut buffer = vec![0; data.len()]; + buffer.copy_from_slice(&data); + buffer.into_boxed_slice() + } +} + /// Represents an instruction to read a known buffer. #[derive(Debug)] pub struct BufferSliceInstruction { diff --git a/src/common/node.rs b/src/common/node.rs index ba1fc416..a04a2903 100644 --- a/src/common/node.rs +++ b/src/common/node.rs @@ -1,5 +1,8 @@ +#[cfg(not(feature = "v10"))] // tree module has the byte manipulation for v10 use anyhow::ensure; +#[cfg(not(feature = "v10"))] // tree module has the byte manipulation for v10 use anyhow::Result; +#[cfg(not(feature = "v10"))] // tree module has the byte manipulation for v10 use byteorder::{BigEndian, ReadBytesExt, WriteBytesExt}; use merkle_tree_stream::Node as NodeTrait; use merkle_tree_stream::{NodeKind, NodeParts}; @@ -7,6 +10,7 @@ use pretty_hash::fmt as pretty_fmt; use std::cmp::Ordering; use std::convert::AsRef; use std::fmt::{self, Display}; +#[cfg(not(feature = "v10"))] // tree module has the byte manipulation for v10 use std::io::Cursor; use crate::crypto::Hash; @@ -40,6 +44,7 @@ impl Node { /// Convert a vector to a new instance. /// /// Requires the index at which the buffer was read to be passed. + #[cfg(not(feature = "v10"))] // tree module has the byte manipulation for v10 pub fn from_bytes(index: u64, buffer: &[u8]) -> Result { ensure!(buffer.len() == 40, "buffer should be 40 bytes"); @@ -64,6 +69,7 @@ impl Node { } /// Convert to a buffer that can be written to disk. + #[cfg(not(feature = "v10"))] // tree module has the byte manipulation for v10 pub fn to_bytes(&self) -> Result> { let mut writer = Vec::with_capacity(40); writer.extend_from_slice(&self.hash); diff --git a/src/compact_encoding/mod.rs b/src/compact_encoding/mod.rs index 73ea4318..84902b50 100644 --- a/src/compact_encoding/mod.rs +++ b/src/compact_encoding/mod.rs @@ -37,7 +37,7 @@ impl State { } /// Create a state from existing buffer. - pub fn from_buffer(buffer: &Box<[u8]>) -> State { + pub fn from_buffer(buffer: &[u8]) -> State { State::new_with_start_and_end(0, buffer.len()) } @@ -62,7 +62,7 @@ impl State { } /// Decode a String - pub fn decode_string(&mut self, buffer: &Box<[u8]>) -> String { + pub fn decode_string(&mut self, buffer: &[u8]) -> String { let len = self.decode_usize_var(buffer); let value = std::str::from_utf8(&buffer[self.start..self.start + len]) .expect("string is invalid UTF-8"); @@ -84,7 +84,7 @@ impl State { } /// Decode a fixed length u16 - pub fn decode_u16(&mut self, buffer: &Box<[u8]>) -> u16 { + pub fn decode_u16(&mut self, buffer: &[u8]) -> u16 { let value: u16 = ((buffer[self.start] as u16) << 0) | ((buffer[self.start + 1] as u16) << 8); self.start += 2; @@ -114,7 +114,7 @@ impl State { } /// Decode a variable length u32 - pub fn decode_u32_var(&mut self, buffer: &Box<[u8]>) -> u32 { + pub fn decode_u32_var(&mut self, buffer: &[u8]) -> u32 { let first = buffer[self.start]; self.start += 1; if first < U16_SIGNIFIER { @@ -127,7 +127,7 @@ impl State { } /// Decode a fixed length u32 - pub fn decode_u32(&mut self, buffer: &Box<[u8]>) -> u32 { + pub fn decode_u32(&mut self, buffer: &[u8]) -> u32 { let value: u32 = ((buffer[self.start] as u32) << 0) | ((buffer[self.start + 1] as u32) << 8) | ((buffer[self.start + 2] as u32) << 16) @@ -163,7 +163,7 @@ impl State { } /// Decode a variable length u64 - pub fn decode_u64_var(&mut self, buffer: &Box<[u8]>) -> u64 { + pub fn decode_u64_var(&mut self, buffer: &[u8]) -> u64 { let first = buffer[self.start]; self.start += 1; if first < U16_SIGNIFIER { @@ -178,7 +178,7 @@ impl State { } /// Decode a fixed length u64 - pub fn decode_u64(&mut self, buffer: &Box<[u8]>) -> u64 { + pub fn decode_u64(&mut self, buffer: &[u8]) -> u64 { let value: u64 = ((buffer[self.start] as u64) << 0) | ((buffer[self.start + 1] as u64) << 8) | ((buffer[self.start + 2] as u64) << 16) @@ -207,7 +207,7 @@ impl State { } /// Decode a byte buffer - pub fn decode_buffer(&mut self, buffer: &Box<[u8]>) -> Box<[u8]> { + pub fn decode_buffer(&mut self, buffer: &[u8]) -> Box<[u8]> { let len = self.decode_usize_var(buffer); let value = buffer[self.start..self.start + len] .to_vec() @@ -235,7 +235,7 @@ impl State { } /// Decode a String array - pub fn decode_string_array(&mut self, buffer: &Box<[u8]>) -> Vec { + pub fn decode_string_array(&mut self, buffer: &[u8]) -> Vec { let len = self.decode_usize_var(buffer); let mut value = Vec::with_capacity(len); for _ in 0..len { @@ -299,7 +299,7 @@ impl State { } } - fn decode_usize_var(&mut self, buffer: &Box<[u8]>) -> usize { + fn decode_usize_var(&mut self, buffer: &[u8]) -> usize { let first = buffer[self.start]; self.start += 1; // NB: the from_le_bytes needs a [u8; 2] and that can't be efficiently @@ -330,7 +330,7 @@ where fn encode(&mut self, value: &T, buffer: &mut Box<[u8]>); /// Decode - fn decode(&mut self, buffer: &Box<[u8]>) -> T; + fn decode(&mut self, buffer: &[u8]) -> T; } impl CompactEncoding for State { @@ -342,7 +342,7 @@ impl CompactEncoding for State { self.encode_str(value, buffer) } - fn decode(&mut self, buffer: &Box<[u8]>) -> String { + fn decode(&mut self, buffer: &[u8]) -> String { self.decode_string(buffer) } } @@ -356,7 +356,7 @@ impl CompactEncoding for State { self.encode_u32_var(value, buffer) } - fn decode(&mut self, buffer: &Box<[u8]>) -> u32 { + fn decode(&mut self, buffer: &[u8]) -> u32 { self.decode_u32_var(buffer) } } @@ -370,7 +370,7 @@ impl CompactEncoding for State { self.encode_u64_var(value, buffer) } - fn decode(&mut self, buffer: &Box<[u8]>) -> u64 { + fn decode(&mut self, buffer: &[u8]) -> u64 { self.decode_u64_var(buffer) } } @@ -384,7 +384,7 @@ impl CompactEncoding for State { self.encode_usize_var(value, buffer) } - fn decode(&mut self, buffer: &Box<[u8]>) -> usize { + fn decode(&mut self, buffer: &[u8]) -> usize { self.decode_usize_var(buffer) } } @@ -398,7 +398,7 @@ impl CompactEncoding> for State { self.encode_buffer(value, buffer); } - fn decode(&mut self, buffer: &Box<[u8]>) -> Box<[u8]> { + fn decode(&mut self, buffer: &[u8]) -> Box<[u8]> { self.decode_buffer(buffer) } } @@ -412,7 +412,7 @@ impl CompactEncoding> for State { self.encode_string_array(value, buffer); } - fn decode(&mut self, buffer: &Box<[u8]>) -> Vec { + fn decode(&mut self, buffer: &[u8]) -> Vec { self.decode_string_array(buffer) } } diff --git a/src/core.rs b/src/core.rs index a1f7fea8..b0a7c0fa 100644 --- a/src/core.rs +++ b/src/core.rs @@ -58,7 +58,7 @@ where .await?; // Open/create tree - let header_tree_length = oplog_open_outcome.header.tree.length; + let header_tree_length = &oplog_open_outcome.header.tree.length; let slice_instructions = MerkleTree::get_slice_instructions_to_read(header_tree_length); let slices = storage.read_slices(Store::Tree, slice_instructions).await?; let tree = MerkleTree::open(header_tree_length, slices)?; diff --git a/src/oplog/mod.rs b/src/oplog/mod.rs index 38c7a8cf..3d1257cd 100644 --- a/src/oplog/mod.rs +++ b/src/oplog/mod.rs @@ -78,7 +78,7 @@ impl CompactEncoding for State { self.encode(&value.signer, buffer); } - fn decode(&mut self, buffer: &Box<[u8]>) -> HeaderTypes { + fn decode(&mut self, buffer: &[u8]) -> HeaderTypes { let tree: String = self.decode(buffer); let bitfield: String = self.decode(buffer); let signer: String = self.decode(buffer); @@ -125,7 +125,7 @@ impl CompactEncoding for State { self.encode(&value.signature, buffer); } - fn decode(&mut self, buffer: &Box<[u8]>) -> HeaderTree { + fn decode(&mut self, buffer: &[u8]) -> HeaderTree { let fork: u64 = self.decode(buffer); let length: u64 = self.decode(buffer); let root_hash: Box<[u8]> = self.decode(buffer); @@ -173,7 +173,7 @@ impl CompactEncoding for State { } } - fn decode(&mut self, buffer: &Box<[u8]>) -> PartialKeypair { + fn decode(&mut self, buffer: &[u8]) -> PartialKeypair { let public_key_bytes: Box<[u8]> = self.decode(buffer); let secret_key_bytes: Box<[u8]> = self.decode(buffer); let secret: Option = if secret_key_bytes.len() == 0 { @@ -204,7 +204,7 @@ impl CompactEncoding for State { self.encode(&value.reorgs, buffer); } - fn decode(&mut self, buffer: &Box<[u8]>) -> HeaderHints { + fn decode(&mut self, buffer: &[u8]) -> HeaderHints { HeaderHints { reorgs: self.decode(buffer), } @@ -233,7 +233,7 @@ impl CompactEncoding
for State { self.encode(&value.contiguous_length, buffer); } - fn decode(&mut self, buffer: &Box<[u8]>) -> Header { + fn decode(&mut self, buffer: &[u8]) -> Header { let version: u8 = buffer[self.start]; self.start += 1; if version != 0 { diff --git a/src/tree/merkle_tree.rs b/src/tree/merkle_tree.rs index 05f69b5c..a5049b55 100644 --- a/src/tree/merkle_tree.rs +++ b/src/tree/merkle_tree.rs @@ -1,34 +1,72 @@ +use anyhow::ensure; use anyhow::Result; -use crate::common::{BufferSlice, BufferSliceInstruction}; +use crate::compact_encoding::State; +use crate::{ + common::{BufferSlice, BufferSliceInstruction}, + Node, +}; /// Merkle tree. /// See https://github.com/hypercore-protocol/hypercore/blob/master/lib/merkle-tree.js #[derive(Debug)] -pub struct MerkleTree {} +pub struct MerkleTree { + roots: Vec, +} + +const NODE_SIZE: u64 = 40; impl MerkleTree { /// Gets instructions to slices that should be read from storage /// based on `HeaderTree` `length` field. Call this before /// calling `open`. pub fn get_slice_instructions_to_read( - header_tree_length: u64, + header_tree_length: &u64, ) -> Box<[BufferSliceInstruction]> { - let mut roots = vec![]; - flat_tree::full_roots(header_tree_length * 2, &mut roots); + let roots = get_root_indices(header_tree_length); roots .iter() - .map(|&index| BufferSliceInstruction { index, len: 8 }) + .map(|&index| BufferSliceInstruction { + index: NODE_SIZE * index, + len: NODE_SIZE, + }) .collect::>() .into_boxed_slice() } /// Opens MerkleTree, based on read byte slices. Call `get_slice_instructions_to_read` - /// before calling this to find out which slices to read. - pub fn open(header_tree_length: u64, slices: Box<[BufferSlice]>) -> Result { - // read_tree_bytes(0, 0).await?; - // read_tree_bytes(0, 0)?; - Ok(Self {}) + /// before calling this to find out which slices to read. The given slices + /// need to be in the same order as the instructions from `get_slice_instructions_to_read`! + pub fn open(header_tree_length: &u64, slices: Box<[BufferSlice]>) -> Result { + let roots = get_root_indices(header_tree_length); + + let mut nodes: Vec = Vec::with_capacity(slices.len()); + for i in 0..roots.len() { + let index = roots[i]; + ensure!( + index == slices[i].index / NODE_SIZE, + "Given slices vector not in the correct order" + ); + let data = slices[i].data.as_ref().unwrap(); + let node = node_from_bytes(&index, data); + nodes.push(node); + } + + Ok(Self { roots: nodes }) } } + +fn get_root_indices(header_tree_length: &u64) -> Vec { + let mut roots = vec![]; + flat_tree::full_roots(header_tree_length * 2, &mut roots); + roots +} + +fn node_from_bytes(index: &u64, data: &[u8]) -> Node { + let len_buf = &data[..8]; + let hash = &data[8..]; + let mut state = State::from_buffer(len_buf); + let len = state.decode_u64(len_buf); + Node::new(*index, hash.to_vec(), len) +} From 68fa704b32b16042efa4a7c96903a8a8d5222312 Mon Sep 17 00:00:00 2001 From: Timo Tiuraniemi Date: Fri, 2 Sep 2022 10:21:23 +0300 Subject: [PATCH 029/157] Function params don't need Box --- src/compact_encoding/mod.rs | 36 ++++++++++++++++++------------------ src/oplog/mod.rs | 10 +++++----- 2 files changed, 23 insertions(+), 23 deletions(-) diff --git a/src/compact_encoding/mod.rs b/src/compact_encoding/mod.rs index 84902b50..906b2f41 100644 --- a/src/compact_encoding/mod.rs +++ b/src/compact_encoding/mod.rs @@ -54,7 +54,7 @@ impl State { } /// Encode a string slice - pub fn encode_str(&mut self, value: &str, buffer: &mut Box<[u8]>) { + pub fn encode_str(&mut self, value: &str, buffer: &mut [u8]) { let len = value.len(); self.encode_usize_var(&len, buffer); buffer[self.start..self.start + len].copy_from_slice(value.as_bytes()); @@ -92,7 +92,7 @@ impl State { } /// Encode a variable length u32 - pub fn encode_u32_var(&mut self, value: &u32, buffer: &mut Box<[u8]>) { + pub fn encode_u32_var(&mut self, value: &u32, buffer: &mut [u8]) { if *value < U16_SIGNIFIER.into() { let bytes = value.to_le_bytes(); buffer[self.start] = bytes[0]; @@ -109,7 +109,7 @@ impl State { } /// Encode u32 to 4 LE bytes. - pub fn encode_u32(&mut self, uint: u32, buffer: &mut Box<[u8]>) { + pub fn encode_u32(&mut self, uint: u32, buffer: &mut [u8]) { self.encode_uint32_bytes(&uint.to_le_bytes(), buffer); } @@ -137,7 +137,7 @@ impl State { } /// Encode a variable length u64 - pub fn encode_u64_var(&mut self, value: &u64, buffer: &mut Box<[u8]>) { + pub fn encode_u64_var(&mut self, value: &u64, buffer: &mut [u8]) { if *value < U16_SIGNIFIER.into() { let bytes = value.to_le_bytes(); buffer[self.start] = bytes[0]; @@ -158,7 +158,7 @@ impl State { } /// Encode u64 to 8 LE bytes. - pub fn encode_u64(&mut self, uint: u64, buffer: &mut Box<[u8]>) { + pub fn encode_u64(&mut self, uint: u64, buffer: &mut [u8]) { self.encode_uint64_bytes(&uint.to_le_bytes(), buffer); } @@ -199,7 +199,7 @@ impl State { } /// Encode a byte buffer - pub fn encode_buffer(&mut self, value: &Box<[u8]>, buffer: &mut Box<[u8]>) { + pub fn encode_buffer(&mut self, value: &Box<[u8]>, buffer: &mut [u8]) { let len = value.len(); self.encode_usize_var(&len, buffer); buffer[self.start..self.start + len].copy_from_slice(value); @@ -226,7 +226,7 @@ impl State { } /// Encode a String array - pub fn encode_string_array(&mut self, value: &Vec, buffer: &mut Box<[u8]>) { + pub fn encode_string_array(&mut self, value: &Vec, buffer: &mut [u8]) { let len = value.len(); self.encode_usize_var(&len, buffer); for string_value in value { @@ -244,20 +244,20 @@ impl State { value } - fn encode_uint16_bytes(&mut self, bytes: &[u8], buffer: &mut Box<[u8]>) { + fn encode_uint16_bytes(&mut self, bytes: &[u8], buffer: &mut [u8]) { buffer[self.start] = bytes[0]; buffer[self.start + 1] = bytes[1]; self.start += 2; } - fn encode_uint32_bytes(&mut self, bytes: &[u8], buffer: &mut Box<[u8]>) { + fn encode_uint32_bytes(&mut self, bytes: &[u8], buffer: &mut [u8]) { self.encode_uint16_bytes(bytes, buffer); buffer[self.start] = bytes[2]; buffer[self.start + 1] = bytes[3]; self.start += 2; } - fn encode_uint64_bytes(&mut self, bytes: &[u8], buffer: &mut Box<[u8]>) { + fn encode_uint64_bytes(&mut self, bytes: &[u8], buffer: &mut [u8]) { self.encode_uint32_bytes(bytes, buffer); buffer[self.start] = bytes[4]; buffer[self.start + 1] = bytes[5]; @@ -279,7 +279,7 @@ impl State { }; } - fn encode_usize_var(&mut self, value: &usize, buffer: &mut Box<[u8]>) { + fn encode_usize_var(&mut self, value: &usize, buffer: &mut [u8]) { if *value <= 0xfc { let bytes = value.to_le_bytes(); buffer[self.start] = bytes[0]; @@ -327,7 +327,7 @@ where fn preencode(&mut self, value: &T); /// Encode - fn encode(&mut self, value: &T, buffer: &mut Box<[u8]>); + fn encode(&mut self, value: &T, buffer: &mut [u8]); /// Decode fn decode(&mut self, buffer: &[u8]) -> T; @@ -338,7 +338,7 @@ impl CompactEncoding for State { self.preencode_str(value) } - fn encode(&mut self, value: &String, buffer: &mut Box<[u8]>) { + fn encode(&mut self, value: &String, buffer: &mut [u8]) { self.encode_str(value, buffer) } @@ -352,7 +352,7 @@ impl CompactEncoding for State { self.preencode_uint_var(value) } - fn encode(&mut self, value: &u32, buffer: &mut Box<[u8]>) { + fn encode(&mut self, value: &u32, buffer: &mut [u8]) { self.encode_u32_var(value, buffer) } @@ -366,7 +366,7 @@ impl CompactEncoding for State { self.preencode_uint_var(value) } - fn encode(&mut self, value: &u64, buffer: &mut Box<[u8]>) { + fn encode(&mut self, value: &u64, buffer: &mut [u8]) { self.encode_u64_var(value, buffer) } @@ -380,7 +380,7 @@ impl CompactEncoding for State { self.preencode_usize_var(value) } - fn encode(&mut self, value: &usize, buffer: &mut Box<[u8]>) { + fn encode(&mut self, value: &usize, buffer: &mut [u8]) { self.encode_usize_var(value, buffer) } @@ -394,7 +394,7 @@ impl CompactEncoding> for State { self.preencode_buffer(value); } - fn encode(&mut self, value: &Box<[u8]>, buffer: &mut Box<[u8]>) { + fn encode(&mut self, value: &Box<[u8]>, buffer: &mut [u8]) { self.encode_buffer(value, buffer); } @@ -408,7 +408,7 @@ impl CompactEncoding> for State { self.preencode_string_array(value); } - fn encode(&mut self, value: &Vec, buffer: &mut Box<[u8]>) { + fn encode(&mut self, value: &Vec, buffer: &mut [u8]) { self.encode_string_array(value, buffer); } diff --git a/src/oplog/mod.rs b/src/oplog/mod.rs index 3d1257cd..c954a690 100644 --- a/src/oplog/mod.rs +++ b/src/oplog/mod.rs @@ -72,7 +72,7 @@ impl CompactEncoding for State { self.preencode(&value.signer); } - fn encode(&mut self, value: &HeaderTypes, buffer: &mut Box<[u8]>) { + fn encode(&mut self, value: &HeaderTypes, buffer: &mut [u8]) { self.encode(&value.tree, buffer); self.encode(&value.bitfield, buffer); self.encode(&value.signer, buffer); @@ -118,7 +118,7 @@ impl CompactEncoding for State { self.preencode(&value.signature); } - fn encode(&mut self, value: &HeaderTree, buffer: &mut Box<[u8]>) { + fn encode(&mut self, value: &HeaderTree, buffer: &mut [u8]) { self.encode(&value.fork, buffer); self.encode(&value.length, buffer); self.encode(&value.root_hash, buffer); @@ -155,7 +155,7 @@ impl CompactEncoding for State { } } - fn encode(&mut self, value: &PartialKeypair, buffer: &mut Box<[u8]>) { + fn encode(&mut self, value: &PartialKeypair, buffer: &mut [u8]) { let public_key_bytes: Box<[u8]> = value.public.as_bytes().to_vec().into_boxed_slice(); self.encode(&public_key_bytes, buffer); match &value.secret { @@ -200,7 +200,7 @@ impl CompactEncoding for State { self.preencode(&value.reorgs); } - fn encode(&mut self, value: &HeaderHints, buffer: &mut Box<[u8]>) { + fn encode(&mut self, value: &HeaderHints, buffer: &mut [u8]) { self.encode(&value.reorgs, buffer); } @@ -222,7 +222,7 @@ impl CompactEncoding
for State { self.preencode(&value.contiguous_length); } - fn encode(&mut self, value: &Header, buffer: &mut Box<[u8]>) { + fn encode(&mut self, value: &Header, buffer: &mut [u8]) { buffer[self.start] = 0; // Version self.start += 1; self.encode(&value.types, buffer); From 82a01605bfd7b6b8fa376eb1f1ee2c0c6bc326a8 Mon Sep 17 00:00:00 2001 From: Timo Tiuraniemi Date: Fri, 2 Sep 2022 10:23:11 +0300 Subject: [PATCH 030/157] Same with encode_buffer, does not need Box --- src/compact_encoding/mod.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/compact_encoding/mod.rs b/src/compact_encoding/mod.rs index 906b2f41..6aa35893 100644 --- a/src/compact_encoding/mod.rs +++ b/src/compact_encoding/mod.rs @@ -199,7 +199,7 @@ impl State { } /// Encode a byte buffer - pub fn encode_buffer(&mut self, value: &Box<[u8]>, buffer: &mut [u8]) { + pub fn encode_buffer(&mut self, value: &[u8], buffer: &mut [u8]) { let len = value.len(); self.encode_usize_var(&len, buffer); buffer[self.start..self.start + len].copy_from_slice(value); From 9725ca43842b3cfe43b48d9ecf77d2cbf776cc20 Mon Sep 17 00:00:00 2001 From: Timo Tiuraniemi Date: Fri, 2 Sep 2022 11:05:29 +0300 Subject: [PATCH 031/157] Added byte lenght and total length to the MerkleTree --- src/tree/merkle_tree.rs | 28 ++++++++++++++++++++-------- 1 file changed, 20 insertions(+), 8 deletions(-) diff --git a/src/tree/merkle_tree.rs b/src/tree/merkle_tree.rs index a5049b55..8b2b6f2c 100644 --- a/src/tree/merkle_tree.rs +++ b/src/tree/merkle_tree.rs @@ -12,6 +12,8 @@ use crate::{ #[derive(Debug)] pub struct MerkleTree { roots: Vec, + length: u64, + byte_length: u64, } const NODE_SIZE: u64 = 40; @@ -23,9 +25,9 @@ impl MerkleTree { pub fn get_slice_instructions_to_read( header_tree_length: &u64, ) -> Box<[BufferSliceInstruction]> { - let roots = get_root_indices(header_tree_length); + let root_indices = get_root_indices(header_tree_length); - roots + root_indices .iter() .map(|&index| BufferSliceInstruction { index: NODE_SIZE * index, @@ -39,21 +41,31 @@ impl MerkleTree { /// before calling this to find out which slices to read. The given slices /// need to be in the same order as the instructions from `get_slice_instructions_to_read`! pub fn open(header_tree_length: &u64, slices: Box<[BufferSlice]>) -> Result { - let roots = get_root_indices(header_tree_length); + let root_indices = get_root_indices(header_tree_length); - let mut nodes: Vec = Vec::with_capacity(slices.len()); - for i in 0..roots.len() { - let index = roots[i]; + let mut roots: Vec = Vec::with_capacity(slices.len()); + let mut byte_length: u64 = 0; + let mut length: u64 = 0; + for i in 0..root_indices.len() { + let index = root_indices[i]; ensure!( index == slices[i].index / NODE_SIZE, "Given slices vector not in the correct order" ); let data = slices[i].data.as_ref().unwrap(); let node = node_from_bytes(&index, data); - nodes.push(node); + byte_length += node.length; + // This is totalSpan in Javascript + length += 2 * ((node.index - length) + 1); + + roots.push(node); } - Ok(Self { roots: nodes }) + Ok(Self { + roots, + length, + byte_length, + }) } } From 29c170c560977f2639d3fa03915f10a1a68a8273 Mon Sep 17 00:00:00 2001 From: Timo Tiuraniemi Date: Mon, 5 Sep 2022 15:16:19 +0300 Subject: [PATCH 032/157] Some more work on creating the right kind of merkle tree --- src/compact_encoding/mod.rs | 7 +++ src/core.rs | 15 ++++-- src/crypto/hash.rs | 52 ++++++++++++++++++- src/crypto/mod.rs | 2 +- src/tree/merkle_tree.rs | 20 ++++++-- src/tree/merkle_tree_batch.rs | 4 -- src/tree/merkle_tree_changeset.rs | 84 +++++++++++++++++++++++++++++++ src/tree/mod.rs | 4 +- 8 files changed, 172 insertions(+), 16 deletions(-) delete mode 100644 src/tree/merkle_tree_batch.rs create mode 100644 src/tree/merkle_tree_changeset.rs diff --git a/src/compact_encoding/mod.rs b/src/compact_encoding/mod.rs index 6aa35893..04b7788e 100644 --- a/src/compact_encoding/mod.rs +++ b/src/compact_encoding/mod.rs @@ -216,6 +216,13 @@ impl State { value } + /// Encode a raw byte buffer, skipping the length byte + pub fn encode_raw_buffer(&mut self, value: &[u8], buffer: &mut [u8]) { + let len = value.len(); + buffer[self.start..self.start + len].copy_from_slice(value); + self.start += len; + } + /// Preencode a string array pub fn preencode_string_array(&mut self, value: &Vec) { let len = value.len(); diff --git a/src/core.rs b/src/core.rs index b0a7c0fa..cceab09c 100644 --- a/src/core.rs +++ b/src/core.rs @@ -2,7 +2,7 @@ pub use crate::storage_v10::{PartialKeypair, Storage, Store}; -use crate::{crypto::generate_keypair, oplog::Oplog, tree::MerkleTree}; +use crate::{crypto::generate_keypair, oplog::Oplog, sign, tree::MerkleTree}; use anyhow::Result; use random_access_storage::RandomAccess; use std::fmt::Debug; @@ -58,10 +58,10 @@ where .await?; // Open/create tree - let header_tree_length = &oplog_open_outcome.header.tree.length; - let slice_instructions = MerkleTree::get_slice_instructions_to_read(header_tree_length); + let slice_instructions = + MerkleTree::get_slice_instructions_to_read(&oplog_open_outcome.header.tree); let slices = storage.read_slices(Store::Tree, slice_instructions).await?; - let tree = MerkleTree::open(header_tree_length, slices)?; + let tree = MerkleTree::open(&oplog_open_outcome.header.tree, slices)?; Ok(Hypercore { key_pair, @@ -78,6 +78,13 @@ where None => anyhow::bail!("No secret key, cannot append."), }; + let mut changeset = self.tree.changeset(); + for data in batch { + changeset.append(data); + } + let hash = changeset.hash_and_sign(&self.key_pair.public, &secret_key); + println!("HASH {:?} SIGNATURE {:?}", hash, changeset.signature); + Ok(AppendOutcome { length: 0, byte_length: 0, diff --git a/src/crypto/hash.rs b/src/crypto/hash.rs index 6b820911..1b69bcf5 100644 --- a/src/crypto/hash.rs +++ b/src/crypto/hash.rs @@ -1,7 +1,7 @@ pub use blake2_rfc::blake2b::Blake2bResult; use crate::common::Node; -use crate::compact_encoding::{CompactEncoding, State}; +use crate::compact_encoding::State; use blake2_rfc::blake2b::Blake2b; use byteorder::{BigEndian, WriteBytesExt}; use ed25519_dalek::PublicKey; @@ -16,6 +16,21 @@ const PARENT_TYPE: [u8; 1] = [0x01]; const ROOT_TYPE: [u8; 1] = [0x02]; const HYPERCORE: [u8; 9] = *b"hypercore"; +// These the output of, see `hash_namespace` test below for how they are produced +// https://github.com/hypercore-protocol/hypercore/blob/70b271643c4e4b1e5ecae5bb579966dfe6361ff3/lib/caps.js#L9 +const TREE: [u8; 32] = [ + 0x9F, 0xAC, 0x70, 0xB5, 0xC, 0xA1, 0x4E, 0xFC, 0x4E, 0x91, 0xC8, 0x33, 0xB2, 0x4, 0xE7, 0x5B, + 0x8B, 0x5A, 0xAD, 0x8B, 0x58, 0x81, 0xBF, 0xC0, 0xAD, 0xB5, 0xEF, 0x38, 0xA3, 0x27, 0x5B, 0x9C, +]; +const REPLICATE_INITIATOR: [u8; 32] = [ + 0x51, 0x81, 0x2A, 0x2A, 0x35, 0x9B, 0x50, 0x36, 0x95, 0x36, 0x77, 0x5D, 0xF8, 0x9E, 0x18, 0xE4, + 0x77, 0x40, 0xF3, 0xDB, 0x72, 0xAC, 0xA, 0xE7, 0xB, 0x29, 0x59, 0x4C, 0x19, 0x4D, 0xC3, 0x16, +]; +const REPLICATE_RESPONDER: [u8; 32] = [ + 0x4, 0x38, 0x49, 0x2D, 0x2, 0x97, 0xC, 0xC1, 0x35, 0x28, 0xAC, 0x2, 0x62, 0xBC, 0xA0, 0x7, + 0x4E, 0x9, 0x26, 0x26, 0x2, 0x56, 0x86, 0x5A, 0xCC, 0xC0, 0xBF, 0x15, 0xBD, 0x79, 0x12, 0x7D, +]; + /// `BLAKE2b` hash. #[derive(Debug, Clone, PartialEq)] pub struct Hash { @@ -174,6 +189,16 @@ impl DerefMut for Hash { } } +/// Create a signable buffer for tree. This is treeSignable in Javascript. +pub fn signable_tree(hash: &[u8], length: u64, fork: u64) -> Box<[u8]> { + let (mut state, mut buffer) = State::new_with_size(80); + state.encode_raw_buffer(&TREE, &mut buffer); + state.encode_raw_buffer(&hash, &mut buffer); + state.encode_u64(length, &mut buffer); + state.encode_u64(fork, &mut buffer); + buffer +} + #[cfg(test)] mod tests { use super::*; @@ -181,6 +206,14 @@ mod tests { use self::data_encoding::HEXLOWER; use data_encoding; + fn hash_with_extra_byte(data: &[u8], byte: u8) -> Box<[u8]> { + let mut hasher = Blake2b::new(32); + hasher.update(&data); + hasher.update(&[byte]); + let hash = hasher.finalize(); + hash.as_bytes().into() + } + fn hex_bytes(hex: &str) -> Vec { HEXLOWER.decode(hex.as_bytes()).unwrap() } @@ -284,4 +317,21 @@ mod tests { "0e576a56b478cddb6ffebab8c494532b6de009466b2e9f7af9143fc54b9eaa36", ); } + + // This is the rust version from + // https://github.com/hypercore-protocol/hypercore/blob/70b271643c4e4b1e5ecae5bb579966dfe6361ff3/lib/caps.js + // and validates that our arrays match + #[test] + fn hash_namespace() { + let mut hasher = Blake2b::new(32); + hasher.update(&HYPERCORE); + let hash = hasher.finalize(); + let ns = hash.as_bytes(); + let tree: Box<[u8]> = { hash_with_extra_byte(ns, 0).into() }; + assert_eq!(tree, TREE.into()); + let replicate_initiator: Box<[u8]> = { hash_with_extra_byte(ns, 1).into() }; + assert_eq!(replicate_initiator, REPLICATE_INITIATOR.into()); + let replicate_responder: Box<[u8]> = { hash_with_extra_byte(ns, 2).into() }; + assert_eq!(replicate_responder, REPLICATE_RESPONDER.into()); + } } diff --git a/src/crypto/mod.rs b/src/crypto/mod.rs index e6c10c65..abfe4397 100644 --- a/src/crypto/mod.rs +++ b/src/crypto/mod.rs @@ -4,7 +4,7 @@ mod hash; mod key_pair; mod merkle; -pub use self::hash::Hash; +pub use self::hash::{signable_tree, Hash}; pub use self::key_pair::{ generate as generate_keypair, sign, verify, PublicKey, SecretKey, Signature, }; diff --git a/src/tree/merkle_tree.rs b/src/tree/merkle_tree.rs index 8b2b6f2c..61bb88f0 100644 --- a/src/tree/merkle_tree.rs +++ b/src/tree/merkle_tree.rs @@ -2,11 +2,14 @@ use anyhow::ensure; use anyhow::Result; use crate::compact_encoding::State; +use crate::oplog::HeaderTree; use crate::{ common::{BufferSlice, BufferSliceInstruction}, Node, }; +use super::MerkleTreeChangeset; + /// Merkle tree. /// See https://github.com/hypercore-protocol/hypercore/blob/master/lib/merkle-tree.js #[derive(Debug)] @@ -14,6 +17,7 @@ pub struct MerkleTree { roots: Vec, length: u64, byte_length: u64, + fork: u64, } const NODE_SIZE: u64 = 40; @@ -23,9 +27,9 @@ impl MerkleTree { /// based on `HeaderTree` `length` field. Call this before /// calling `open`. pub fn get_slice_instructions_to_read( - header_tree_length: &u64, + header_tree: &HeaderTree, ) -> Box<[BufferSliceInstruction]> { - let root_indices = get_root_indices(header_tree_length); + let root_indices = get_root_indices(&header_tree.length); root_indices .iter() @@ -40,8 +44,8 @@ impl MerkleTree { /// Opens MerkleTree, based on read byte slices. Call `get_slice_instructions_to_read` /// before calling this to find out which slices to read. The given slices /// need to be in the same order as the instructions from `get_slice_instructions_to_read`! - pub fn open(header_tree_length: &u64, slices: Box<[BufferSlice]>) -> Result { - let root_indices = get_root_indices(header_tree_length); + pub fn open(header_tree: &HeaderTree, slices: Box<[BufferSlice]>) -> Result { + let root_indices = get_root_indices(&header_tree.length); let mut roots: Vec = Vec::with_capacity(slices.len()); let mut byte_length: u64 = 0; @@ -65,8 +69,16 @@ impl MerkleTree { roots, length, byte_length, + fork: header_tree.fork, }) } + + /// Initialize a changeset for this tree. + /// This is called batch() in Javascript, see: + /// https://github.com/hypercore-protocol/hypercore/blob/master/lib/merkle-tree.js + pub fn changeset(&self) -> MerkleTreeChangeset { + MerkleTreeChangeset::new(self.length, self.byte_length, self.fork, self.roots.clone()) + } } fn get_root_indices(header_tree_length: &u64) -> Vec { diff --git a/src/tree/merkle_tree_batch.rs b/src/tree/merkle_tree_batch.rs deleted file mode 100644 index ed9fb5ef..00000000 --- a/src/tree/merkle_tree_batch.rs +++ /dev/null @@ -1,4 +0,0 @@ -/// Merkle tree batch. -/// See https://github.com/hypercore-protocol/hypercore/blob/master/lib/merkle-tree.js -#[derive(Debug)] -pub struct MerkleTreeBatch {} diff --git a/src/tree/merkle_tree_changeset.rs b/src/tree/merkle_tree_changeset.rs new file mode 100644 index 00000000..945edec0 --- /dev/null +++ b/src/tree/merkle_tree_changeset.rs @@ -0,0 +1,84 @@ +use ed25519_dalek::{PublicKey, SecretKey, Signature}; + +use crate::{ + crypto::{signable_tree, Hash}, + sign, Node, +}; + +/// Changeset for a `MerkleTree`. This allows to incrementally change a `MerkleTree` in two steps: +/// first create the changes to this changeset, get out information from this to put to the oplog, +/// and the commit the changeset to the tree. +/// +/// This is called "MerkleTreeBatch" in Javascript, see: +/// https://github.com/hypercore-protocol/hypercore/blob/master/lib/merkle-tree.js +#[derive(Debug)] +pub struct MerkleTreeChangeset { + pub(crate) length: u64, + pub(crate) byte_length: u64, + pub(crate) fork: u64, + pub(crate) roots: Vec, + pub(crate) nodes: Vec, + pub(crate) signature: Option, +} + +impl MerkleTreeChangeset { + pub fn new(length: u64, byte_length: u64, fork: u64, roots: Vec) -> MerkleTreeChangeset { + Self { + length, + byte_length, + fork, + roots, + nodes: vec![], + signature: None, + } + } + + pub fn append(&mut self, data: &[u8]) { + let len = data.len() as u64; + let head = self.length * 2; + let iter = flat_tree::Iterator::new(head); + let node = Node::new(head, Hash::data(data).as_bytes().to_vec(), len); + self.append_root(node, iter); + } + + pub fn append_root(&mut self, node: Node, iter: flat_tree::Iterator) { + self.length += iter.factor() / 2; + self.byte_length += node.length; + self.roots.push(node.clone()); + self.nodes.push(node); + + // TODO: Javascript has this + // + // while (this.roots.length > 1) { + // const a = this.roots[this.roots.length - 1] + // const b = this.roots[this.roots.length - 2] + + // // TODO: just have a peek sibling instead? (pretty sure it's always the left sib as well) + // if (ite.sibling() !== b.index) { + // ite.sibling() // unset so it always points to last root + // break + // } + + // const node = parentNode(this.tree.crypto, ite.parent(), a, b) + // this.nodes.push(node) + // this.roots.pop() + // this.roots.pop() + // this.roots.push(node) + // } + // } + } + + /// Hashes and signs the changeset + pub fn hash_and_sign(&mut self, public_key: &PublicKey, secret_key: &SecretKey) -> Box<[u8]> { + let hash = self.hash(); + let signable = signable_tree(&hash, self.length, self.fork); + let signature = sign(&public_key, &secret_key, &signable); + self.signature = Some(signature); + hash + } + + /// Calculates a hash of the current set of roots + pub fn hash(&self) -> Box<[u8]> { + Hash::tree(&self.roots).as_bytes().into() + } +} diff --git a/src/tree/mod.rs b/src/tree/mod.rs index 54eb79cb..d1cea59d 100644 --- a/src/tree/mod.rs +++ b/src/tree/mod.rs @@ -1,5 +1,5 @@ mod merkle_tree; -mod merkle_tree_batch; +mod merkle_tree_changeset; pub use merkle_tree::MerkleTree; -pub use merkle_tree_batch::MerkleTreeBatch; +pub use merkle_tree_changeset::MerkleTreeChangeset; From cb9456b20258dcf48c8255d6bd902e493a5dc1b3 Mon Sep 17 00:00:00 2001 From: Timo Tiuraniemi Date: Mon, 5 Sep 2022 15:42:13 +0300 Subject: [PATCH 033/157] Move compact encoding stuff to their own files, initial file for Entry --- src/core.rs | 1 - src/oplog/entry.rs | 26 ++++ src/oplog/header.rs | 333 +++++++++++++++++++++++++++++++++++++++++++ src/oplog/mod.rs | 336 +------------------------------------------- 4 files changed, 363 insertions(+), 333 deletions(-) create mode 100644 src/oplog/entry.rs create mode 100644 src/oplog/header.rs diff --git a/src/core.rs b/src/core.rs index cceab09c..e1a968fa 100644 --- a/src/core.rs +++ b/src/core.rs @@ -83,7 +83,6 @@ where changeset.append(data); } let hash = changeset.hash_and_sign(&self.key_pair.public, &secret_key); - println!("HASH {:?} SIGNATURE {:?}", hash, changeset.signature); Ok(AppendOutcome { length: 0, diff --git a/src/oplog/entry.rs b/src/oplog/entry.rs new file mode 100644 index 00000000..8f70f3ca --- /dev/null +++ b/src/oplog/entry.rs @@ -0,0 +1,26 @@ +use crate::compact_encoding::{CompactEncoding, State}; + +/// Oplog Entry +#[derive(Debug)] +pub struct Entry { + // userData: null, + // treeNodes: batch.nodes, + // treeUpgrade: batch, + // bitfield: { + // drop: false, + // start: batch.ancestors, + // length: values.length + // } + // TODO: This is a keyValueArray in JS + pub(crate) user_data: Vec, +} + +impl CompactEncoding for State { + fn preencode(&mut self, value: &Entry) {} + + fn encode(&mut self, value: &Entry, buffer: &mut [u8]) {} + + fn decode(&mut self, buffer: &[u8]) -> Entry { + Entry { user_data: vec![] } + } +} diff --git a/src/oplog/header.rs b/src/oplog/header.rs new file mode 100644 index 00000000..6bd00961 --- /dev/null +++ b/src/oplog/header.rs @@ -0,0 +1,333 @@ +use crate::compact_encoding::{CompactEncoding, State}; +use crate::crypto::{PublicKey, SecretKey}; +use crate::PartialKeypair; + +/// Oplog header. +#[derive(Debug)] +pub struct Header { + pub(crate) types: HeaderTypes, + // TODO: This is a keyValueArray in JS + pub(crate) user_data: Vec, + pub(crate) tree: HeaderTree, + pub(crate) signer: PartialKeypair, + pub(crate) hints: HeaderHints, + pub(crate) contiguous_length: u64, +} + +impl Header { + /// Creates a new Header from given key pair + pub fn new(key_pair: PartialKeypair) -> Header { + Header { + types: HeaderTypes::new(), + user_data: vec![], + tree: HeaderTree::new(), + signer: key_pair, + hints: HeaderHints { reorgs: vec![] }, + contiguous_length: 0, + } + // Javascript side, initial header + // + // header = { + // types: { tree: 'blake2b', bitfield: 'raw', signer: 'ed25519' }, + // userData: [], + // tree: { + // fork: 0, + // length: 0, + // rootHash: null, + // signature: null + // }, + // signer: opts.keyPair || crypto.keyPair(), + // hints: { + // reorgs: [] + // }, + // contiguousLength: 0 + // } + } +} + +/// Oplog header types +#[derive(Debug, PartialEq)] +pub struct HeaderTypes { + pub(crate) tree: String, + pub(crate) bitfield: String, + pub(crate) signer: String, +} +impl HeaderTypes { + pub fn new() -> HeaderTypes { + HeaderTypes { + tree: "blake2b".to_string(), + bitfield: "raw".to_string(), + signer: "ed25519".to_string(), + } + } +} + +impl CompactEncoding for State { + fn preencode(&mut self, value: &HeaderTypes) { + self.preencode(&value.tree); + self.preencode(&value.bitfield); + self.preencode(&value.signer); + } + + fn encode(&mut self, value: &HeaderTypes, buffer: &mut [u8]) { + self.encode(&value.tree, buffer); + self.encode(&value.bitfield, buffer); + self.encode(&value.signer, buffer); + } + + fn decode(&mut self, buffer: &[u8]) -> HeaderTypes { + let tree: String = self.decode(buffer); + let bitfield: String = self.decode(buffer); + let signer: String = self.decode(buffer); + HeaderTypes { + tree, + bitfield, + signer, + } + } +} + +/// Oplog header tree +#[derive(Debug, PartialEq)] +pub struct HeaderTree { + pub(crate) fork: u64, + pub(crate) length: u64, + pub(crate) root_hash: Box<[u8]>, + pub(crate) signature: Box<[u8]>, +} + +impl HeaderTree { + pub fn new() -> HeaderTree { + HeaderTree { + fork: 0, + length: 0, + root_hash: Box::new([]), + signature: Box::new([]), + } + } +} + +impl CompactEncoding for State { + fn preencode(&mut self, value: &HeaderTree) { + self.preencode(&value.fork); + self.preencode(&value.length); + self.preencode(&value.root_hash); + self.preencode(&value.signature); + } + + fn encode(&mut self, value: &HeaderTree, buffer: &mut [u8]) { + self.encode(&value.fork, buffer); + self.encode(&value.length, buffer); + self.encode(&value.root_hash, buffer); + self.encode(&value.signature, buffer); + } + + fn decode(&mut self, buffer: &[u8]) -> HeaderTree { + let fork: u64 = self.decode(buffer); + let length: u64 = self.decode(buffer); + let root_hash: Box<[u8]> = self.decode(buffer); + let signature: Box<[u8]> = self.decode(buffer); + HeaderTree { + fork, + length, + root_hash, + signature, + } + } +} + +/// NB: In Javascript's sodium the secret key contains in itself also the public key, so to +/// maintain binary compatibility, we store the public key in the oplog now twice. +impl CompactEncoding for State { + fn preencode(&mut self, value: &PartialKeypair) { + self.end += 1 + 32; + match &value.secret { + Some(_) => { + // Also add room for the public key + self.end += 1 + 64; + } + None => { + self.end += 1; + } + } + } + + fn encode(&mut self, value: &PartialKeypair, buffer: &mut [u8]) { + let public_key_bytes: Box<[u8]> = value.public.as_bytes().to_vec().into_boxed_slice(); + self.encode(&public_key_bytes, buffer); + match &value.secret { + Some(secret_key) => { + let mut secret_key_bytes: Vec = Vec::with_capacity(64); + secret_key_bytes.extend_from_slice(secret_key.as_bytes()); + secret_key_bytes.extend_from_slice(&public_key_bytes); + let secret_key_bytes: Box<[u8]> = secret_key_bytes.into_boxed_slice(); + self.encode(&secret_key_bytes, buffer); + } + None => { + buffer[self.start] = 0; + self.start += 1; + } + } + } + + fn decode(&mut self, buffer: &[u8]) -> PartialKeypair { + let public_key_bytes: Box<[u8]> = self.decode(buffer); + let secret_key_bytes: Box<[u8]> = self.decode(buffer); + let secret: Option = if secret_key_bytes.len() == 0 { + None + } else { + Some(SecretKey::from_bytes(&secret_key_bytes[0..32]).unwrap()) + }; + + PartialKeypair { + public: PublicKey::from_bytes(&public_key_bytes).unwrap(), + secret, + } + } +} + +/// Oplog header hints +#[derive(Debug)] +pub struct HeaderHints { + pub(crate) reorgs: Vec, +} + +impl CompactEncoding for State { + fn preencode(&mut self, value: &HeaderHints) { + self.preencode(&value.reorgs); + } + + fn encode(&mut self, value: &HeaderHints, buffer: &mut [u8]) { + self.encode(&value.reorgs, buffer); + } + + fn decode(&mut self, buffer: &[u8]) -> HeaderHints { + HeaderHints { + reorgs: self.decode(buffer), + } + } +} + +impl CompactEncoding
for State { + fn preencode(&mut self, value: &Header) { + self.end += 1; // Version + self.preencode(&value.types); + self.preencode(&value.user_data); + self.preencode(&value.tree); + self.preencode(&value.signer); + self.preencode(&value.hints); + self.preencode(&value.contiguous_length); + } + + fn encode(&mut self, value: &Header, buffer: &mut [u8]) { + buffer[self.start] = 0; // Version + self.start += 1; + self.encode(&value.types, buffer); + self.encode(&value.user_data, buffer); + self.encode(&value.tree, buffer); + self.encode(&value.signer, buffer); + self.encode(&value.hints, buffer); + self.encode(&value.contiguous_length, buffer); + } + + fn decode(&mut self, buffer: &[u8]) -> Header { + let version: u8 = buffer[self.start]; + self.start += 1; + if version != 0 { + panic!("Unknown oplog version {}", version); + } + let types: HeaderTypes = self.decode(buffer); + let user_data: Vec = self.decode(buffer); + let tree: HeaderTree = self.decode(buffer); + let signer: PartialKeypair = self.decode(buffer); + let hints: HeaderHints = self.decode(buffer); + let contiguous_length: u64 = self.decode(buffer); + + Header { + types, + user_data, + tree, + signer, + hints, + contiguous_length, + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + use crate::crypto::generate_keypair; + + #[test] + fn encode_header_types() { + let mut enc_state = State::new_with_start_and_end(8, 8); + let header_types = HeaderTypes::new(); + enc_state.preencode(&header_types); + let mut buffer = enc_state.create_buffer(); + enc_state.encode(&header_types, &mut buffer); + let mut dec_state = State::from_buffer(&buffer); + dec_state.start = 8; + let header_types_ret: HeaderTypes = dec_state.decode(&buffer); + assert_eq!(header_types, header_types_ret); + } + + #[test] + fn encode_partial_key_pair() { + let mut enc_state = State::new(); + let key_pair = generate_keypair(); + let key_pair = PartialKeypair { + public: key_pair.public, + secret: Some(key_pair.secret), + }; + enc_state.preencode(&key_pair); + let mut buffer = enc_state.create_buffer(); + // Pub key: 1 byte for length, 32 bytes for content + // Sec key: 1 byte for length, 64 bytes for data + let expected_len = 1 + 32 + 1 + 64; + assert_eq!(buffer.len(), expected_len); + assert_eq!(enc_state.end, expected_len); + assert_eq!(enc_state.start, 0); + enc_state.encode(&key_pair, &mut buffer); + let mut dec_state = State::from_buffer(&buffer); + let key_pair_ret: PartialKeypair = dec_state.decode(&buffer); + assert_eq!(key_pair.public, key_pair_ret.public); + assert_eq!( + key_pair.secret.unwrap().as_bytes(), + key_pair_ret.secret.unwrap().as_bytes() + ); + } + + #[test] + fn encode_tree() { + let mut enc_state = State::new(); + let tree = HeaderTree::new(); + enc_state.preencode(&tree); + let mut buffer = enc_state.create_buffer(); + enc_state.encode(&tree, &mut buffer); + let mut dec_state = State::from_buffer(&buffer); + let tree_ret: HeaderTree = dec_state.decode(&buffer); + assert_eq!(tree, tree_ret); + } + + #[test] + fn encode_header() { + let mut enc_state = State::new(); + let key_pair = generate_keypair(); + let key_pair = PartialKeypair { + public: key_pair.public, + secret: Some(key_pair.secret), + }; + let header = Header::new(key_pair); + enc_state.preencode(&header); + let mut buffer = enc_state.create_buffer(); + enc_state.encode(&header, &mut buffer); + let mut dec_state = State::from_buffer(&buffer); + let header_ret: Header = dec_state.decode(&buffer); + assert_eq!(header.signer.public, header_ret.signer.public); + assert_eq!(header.tree.fork, header_ret.tree.fork); + assert_eq!(header.tree.length, header_ret.tree.length); + assert_eq!(header.types, header_ret.types); + } +} diff --git a/src/oplog/mod.rs b/src/oplog/mod.rs index c954a690..f27a28a3 100644 --- a/src/oplog/mod.rs +++ b/src/oplog/mod.rs @@ -1,261 +1,14 @@ use crate::common::BufferSlice; use crate::compact_encoding::{CompactEncoding, State}; -use crate::crypto::{PublicKey, SecretKey}; use crate::PartialKeypair; use anyhow::{anyhow, Result}; use std::convert::{TryFrom, TryInto}; -/// Oplog header. -#[derive(Debug)] -pub struct Header { - pub(crate) types: HeaderTypes, - // TODO: This is a keyValueArray in JS - pub(crate) user_data: Vec, - pub(crate) tree: HeaderTree, - pub(crate) signer: PartialKeypair, - pub(crate) hints: HeaderHints, - pub(crate) contiguous_length: u64, -} +mod entry; +mod header; -impl Header { - /// Creates a new Header from given key pair - pub fn new(key_pair: PartialKeypair) -> Header { - Header { - types: HeaderTypes::new(), - user_data: vec![], - tree: HeaderTree::new(), - signer: key_pair, - hints: HeaderHints { reorgs: vec![] }, - contiguous_length: 0, - } - // Javascript side, initial header - // - // header = { - // types: { tree: 'blake2b', bitfield: 'raw', signer: 'ed25519' }, - // userData: [], - // tree: { - // fork: 0, - // length: 0, - // rootHash: null, - // signature: null - // }, - // signer: opts.keyPair || crypto.keyPair(), - // hints: { - // reorgs: [] - // }, - // contiguousLength: 0 - // } - } -} - -/// Oplog header types -#[derive(Debug, PartialEq)] -pub struct HeaderTypes { - pub(crate) tree: String, - pub(crate) bitfield: String, - pub(crate) signer: String, -} -impl HeaderTypes { - pub fn new() -> HeaderTypes { - HeaderTypes { - tree: "blake2b".to_string(), - bitfield: "raw".to_string(), - signer: "ed25519".to_string(), - } - } -} - -impl CompactEncoding for State { - fn preencode(&mut self, value: &HeaderTypes) { - self.preencode(&value.tree); - self.preencode(&value.bitfield); - self.preencode(&value.signer); - } - - fn encode(&mut self, value: &HeaderTypes, buffer: &mut [u8]) { - self.encode(&value.tree, buffer); - self.encode(&value.bitfield, buffer); - self.encode(&value.signer, buffer); - } - - fn decode(&mut self, buffer: &[u8]) -> HeaderTypes { - let tree: String = self.decode(buffer); - let bitfield: String = self.decode(buffer); - let signer: String = self.decode(buffer); - HeaderTypes { - tree, - bitfield, - signer, - } - } -} - -/// Oplog header tree -#[derive(Debug, PartialEq)] -pub struct HeaderTree { - pub(crate) fork: u64, - pub(crate) length: u64, - pub(crate) root_hash: Box<[u8]>, - pub(crate) signature: Box<[u8]>, -} - -impl HeaderTree { - pub fn new() -> HeaderTree { - HeaderTree { - fork: 0, - length: 0, - root_hash: Box::new([]), - signature: Box::new([]), - } - } -} - -impl CompactEncoding for State { - fn preencode(&mut self, value: &HeaderTree) { - self.preencode(&value.fork); - self.preencode(&value.length); - self.preencode(&value.root_hash); - self.preencode(&value.signature); - } - - fn encode(&mut self, value: &HeaderTree, buffer: &mut [u8]) { - self.encode(&value.fork, buffer); - self.encode(&value.length, buffer); - self.encode(&value.root_hash, buffer); - self.encode(&value.signature, buffer); - } - - fn decode(&mut self, buffer: &[u8]) -> HeaderTree { - let fork: u64 = self.decode(buffer); - let length: u64 = self.decode(buffer); - let root_hash: Box<[u8]> = self.decode(buffer); - let signature: Box<[u8]> = self.decode(buffer); - HeaderTree { - fork, - length, - root_hash, - signature, - } - } -} - -/// NB: In Javascript's sodium the secret key contains in itself also the public key, so to -/// maintain binary compatibility, we store the public key in the oplog now twice. -impl CompactEncoding for State { - fn preencode(&mut self, value: &PartialKeypair) { - self.end += 1 + 32; - match &value.secret { - Some(_) => { - // Also add room for the public key - self.end += 1 + 64; - } - None => { - self.end += 1; - } - } - } - - fn encode(&mut self, value: &PartialKeypair, buffer: &mut [u8]) { - let public_key_bytes: Box<[u8]> = value.public.as_bytes().to_vec().into_boxed_slice(); - self.encode(&public_key_bytes, buffer); - match &value.secret { - Some(secret_key) => { - let mut secret_key_bytes: Vec = Vec::with_capacity(64); - secret_key_bytes.extend_from_slice(secret_key.as_bytes()); - secret_key_bytes.extend_from_slice(&public_key_bytes); - let secret_key_bytes: Box<[u8]> = secret_key_bytes.into_boxed_slice(); - self.encode(&secret_key_bytes, buffer); - } - None => { - buffer[self.start] = 0; - self.start += 1; - } - } - } - - fn decode(&mut self, buffer: &[u8]) -> PartialKeypair { - let public_key_bytes: Box<[u8]> = self.decode(buffer); - let secret_key_bytes: Box<[u8]> = self.decode(buffer); - let secret: Option = if secret_key_bytes.len() == 0 { - None - } else { - Some(SecretKey::from_bytes(&secret_key_bytes[0..32]).unwrap()) - }; - - PartialKeypair { - public: PublicKey::from_bytes(&public_key_bytes).unwrap(), - secret, - } - } -} - -/// Oplog header hints -#[derive(Debug)] -pub struct HeaderHints { - pub(crate) reorgs: Vec, -} - -impl CompactEncoding for State { - fn preencode(&mut self, value: &HeaderHints) { - self.preencode(&value.reorgs); - } - - fn encode(&mut self, value: &HeaderHints, buffer: &mut [u8]) { - self.encode(&value.reorgs, buffer); - } - - fn decode(&mut self, buffer: &[u8]) -> HeaderHints { - HeaderHints { - reorgs: self.decode(buffer), - } - } -} - -impl CompactEncoding
for State { - fn preencode(&mut self, value: &Header) { - self.end += 1; // Version - self.preencode(&value.types); - self.preencode(&value.user_data); - self.preencode(&value.tree); - self.preencode(&value.signer); - self.preencode(&value.hints); - self.preencode(&value.contiguous_length); - } - - fn encode(&mut self, value: &Header, buffer: &mut [u8]) { - buffer[self.start] = 0; // Version - self.start += 1; - self.encode(&value.types, buffer); - self.encode(&value.user_data, buffer); - self.encode(&value.tree, buffer); - self.encode(&value.signer, buffer); - self.encode(&value.hints, buffer); - self.encode(&value.contiguous_length, buffer); - } - - fn decode(&mut self, buffer: &[u8]) -> Header { - let version: u8 = buffer[self.start]; - self.start += 1; - if version != 0 { - panic!("Unknown oplog version {}", version); - } - let types: HeaderTypes = self.decode(buffer); - let user_data: Vec = self.decode(buffer); - let tree: HeaderTree = self.decode(buffer); - let signer: PartialKeypair = self.decode(buffer); - let hints: HeaderHints = self.decode(buffer); - let contiguous_length: u64 = self.decode(buffer); - - Header { - types, - user_data, - tree, - signer, - hints, - contiguous_length, - } - } -} +pub use entry::Entry; +pub use header::{Header, HeaderTree}; /// Oplog. /// @@ -481,84 +234,3 @@ impl Oplog { } } } - -#[cfg(test)] -mod tests { - use crate::{ - compact_encoding::{CompactEncoding, State}, - crypto::generate_keypair, - oplog::{Header, HeaderTree, HeaderTypes}, - PartialKeypair, - }; - - #[test] - fn encode_header_types() { - let mut enc_state = State::new_with_start_and_end(8, 8); - let header_types = HeaderTypes::new(); - enc_state.preencode(&header_types); - let mut buffer = enc_state.create_buffer(); - enc_state.encode(&header_types, &mut buffer); - let mut dec_state = State::from_buffer(&buffer); - dec_state.start = 8; - let header_types_ret: HeaderTypes = dec_state.decode(&buffer); - assert_eq!(header_types, header_types_ret); - } - - #[test] - fn encode_partial_key_pair() { - let mut enc_state = State::new(); - let key_pair = generate_keypair(); - let key_pair = PartialKeypair { - public: key_pair.public, - secret: Some(key_pair.secret), - }; - enc_state.preencode(&key_pair); - let mut buffer = enc_state.create_buffer(); - // Pub key: 1 byte for length, 32 bytes for content - // Sec key: 1 byte for length, 64 bytes for data - let expected_len = 1 + 32 + 1 + 64; - assert_eq!(buffer.len(), expected_len); - assert_eq!(enc_state.end, expected_len); - assert_eq!(enc_state.start, 0); - enc_state.encode(&key_pair, &mut buffer); - let mut dec_state = State::from_buffer(&buffer); - let key_pair_ret: PartialKeypair = dec_state.decode(&buffer); - assert_eq!(key_pair.public, key_pair_ret.public); - assert_eq!( - key_pair.secret.unwrap().as_bytes(), - key_pair_ret.secret.unwrap().as_bytes() - ); - } - - #[test] - fn encode_tree() { - let mut enc_state = State::new(); - let tree = HeaderTree::new(); - enc_state.preencode(&tree); - let mut buffer = enc_state.create_buffer(); - enc_state.encode(&tree, &mut buffer); - let mut dec_state = State::from_buffer(&buffer); - let tree_ret: HeaderTree = dec_state.decode(&buffer); - assert_eq!(tree, tree_ret); - } - - #[test] - fn encode_header() { - let mut enc_state = State::new(); - let key_pair = generate_keypair(); - let key_pair = PartialKeypair { - public: key_pair.public, - secret: Some(key_pair.secret), - }; - let header = Header::new(key_pair); - enc_state.preencode(&header); - let mut buffer = enc_state.create_buffer(); - enc_state.encode(&header, &mut buffer); - let mut dec_state = State::from_buffer(&buffer); - let header_ret: Header = dec_state.decode(&buffer); - assert_eq!(header.signer.public, header_ret.signer.public); - assert_eq!(header.tree.fork, header_ret.tree.fork); - assert_eq!(header.tree.length, header_ret.tree.length); - assert_eq!(header.types, header_ret.types); - } -} From 76d448b275da7f2da2e32c1938a4e19fe9a2ec96 Mon Sep 17 00:00:00 2001 From: Timo Tiuraniemi Date: Tue, 6 Sep 2022 08:56:12 +0300 Subject: [PATCH 034/157] Initial entry compact encoding --- src/compact_encoding/mod.rs | 30 +++++- src/crypto/hash.rs | 4 +- src/oplog/entry.rs | 203 +++++++++++++++++++++++++++++++++++- 3 files changed, 226 insertions(+), 11 deletions(-) diff --git a/src/compact_encoding/mod.rs b/src/compact_encoding/mod.rs index 04b7788e..10a86aa5 100644 --- a/src/compact_encoding/mod.rs +++ b/src/compact_encoding/mod.rs @@ -83,6 +83,13 @@ impl State { }; } + /// Decode a fixed length u8 + pub fn decode_u8(&mut self, buffer: &[u8]) -> u8 { + let value: u8 = buffer[self.start]; + self.start += 1; + value + } + /// Decode a fixed length u16 pub fn decode_u16(&mut self, buffer: &[u8]) -> u16 { let value: u16 = @@ -216,11 +223,24 @@ impl State { value } - /// Encode a raw byte buffer, skipping the length byte - pub fn encode_raw_buffer(&mut self, value: &[u8], buffer: &mut [u8]) { - let len = value.len(); - buffer[self.start..self.start + len].copy_from_slice(value); - self.start += len; + /// Preencode a fixed 32 byte buffer + pub fn preencode_fixed_32(&mut self) { + self.end += 32; + } + + /// Encode a fixed 32 byte buffer + pub fn encode_fixed_32(&mut self, value: &[u8], buffer: &mut [u8]) { + buffer[self.start..self.start + 32].copy_from_slice(value); + self.start += 32; + } + + /// Encode a fixed 32 byte buffer + pub fn decode_fixed_32(&mut self, buffer: &[u8]) -> Box<[u8]> { + let value = buffer[self.start..self.start + 32] + .to_vec() + .into_boxed_slice(); + self.start += 32; + value } /// Preencode a string array diff --git a/src/crypto/hash.rs b/src/crypto/hash.rs index 1b69bcf5..b5b458d4 100644 --- a/src/crypto/hash.rs +++ b/src/crypto/hash.rs @@ -192,8 +192,8 @@ impl DerefMut for Hash { /// Create a signable buffer for tree. This is treeSignable in Javascript. pub fn signable_tree(hash: &[u8], length: u64, fork: u64) -> Box<[u8]> { let (mut state, mut buffer) = State::new_with_size(80); - state.encode_raw_buffer(&TREE, &mut buffer); - state.encode_raw_buffer(&hash, &mut buffer); + state.encode_fixed_32(&TREE, &mut buffer); + state.encode_fixed_32(&hash, &mut buffer); state.encode_u64(length, &mut buffer); state.encode_u64(fork, &mut buffer); buffer diff --git a/src/oplog/entry.rs b/src/oplog/entry.rs index 8f70f3ca..2aa09674 100644 --- a/src/oplog/entry.rs +++ b/src/oplog/entry.rs @@ -1,4 +1,128 @@ -use crate::compact_encoding::{CompactEncoding, State}; +use crate::{ + compact_encoding::{CompactEncoding, State}, + Node, +}; + +impl CompactEncoding for State { + fn preencode(&mut self, value: &Node) { + self.preencode(&value.index); + self.preencode(&value.length); + self.preencode_fixed_32(); + } + + fn encode(&mut self, value: &Node, buffer: &mut [u8]) { + self.encode(&value.index, buffer); + self.encode(&value.length, buffer); + self.encode_fixed_32(&value.hash, buffer); + } + + fn decode(&mut self, buffer: &[u8]) -> Node { + let index: u64 = self.decode(buffer); + let length: u64 = self.decode(buffer); + let hash: Box<[u8]> = self.decode_fixed_32(buffer); + Node::new(index, hash.to_vec(), length) + } +} + +impl CompactEncoding> for State { + fn preencode(&mut self, value: &Vec) { + let len = value.len(); + self.preencode(&len); + for val in value.into_iter() { + self.preencode(val); + } + } + + fn encode(&mut self, value: &Vec, buffer: &mut [u8]) { + let len = value.len(); + self.encode(&len, buffer); + for val in value { + self.encode(val, buffer); + } + } + + fn decode(&mut self, buffer: &[u8]) -> Vec { + let len: usize = self.decode(buffer); + let mut value = Vec::with_capacity(len); + for _ in 0..len { + value.push(self.decode(buffer)); + } + value + } +} + +/// Entry tree upgrade +#[derive(Debug)] +pub struct EntryTreeUpgrade { + pub(crate) fork: u64, + pub(crate) ancestors: u64, + pub(crate) length: u64, + pub(crate) signature: Box<[u8]>, +} + +impl CompactEncoding for State { + fn preencode(&mut self, value: &EntryTreeUpgrade) { + self.preencode(&value.fork); + self.preencode(&value.ancestors); + self.preencode(&value.length); + self.preencode(&value.signature); + } + + fn encode(&mut self, value: &EntryTreeUpgrade, buffer: &mut [u8]) { + self.encode(&value.fork, buffer); + self.encode(&value.ancestors, buffer); + self.encode(&value.length, buffer); + self.encode(&value.signature, buffer); + } + + fn decode(&mut self, buffer: &[u8]) -> EntryTreeUpgrade { + let fork: u64 = self.decode(buffer); + let ancestors: u64 = self.decode(buffer); + let length: u64 = self.decode(buffer); + let signature: Box<[u8]> = self.decode(buffer); + EntryTreeUpgrade { + fork, + ancestors, + length, + signature, + } + } +} + +/// Entry bitfield update +#[derive(Debug)] +pub struct EntryBitfieldUpdate { + pub(crate) drop: bool, + pub(crate) start: u64, + pub(crate) length: u64, +} + +impl CompactEncoding for State { + fn preencode(&mut self, value: &EntryBitfieldUpdate) { + self.end += 1; + self.preencode(&value.start); + self.preencode(&value.length); + } + + fn encode(&mut self, value: &EntryBitfieldUpdate, buffer: &mut [u8]) { + let flags: u8 = if value.drop { 1 } else { 0 }; + buffer[self.start] = flags; + self.start += 1; + self.encode(&value.start, buffer); + self.encode(&value.length, buffer); + } + + fn decode(&mut self, buffer: &[u8]) -> EntryBitfieldUpdate { + let flags = self.decode_u8(buffer); + let start: u64 = self.decode(buffer); + let length: u64 = self.decode(buffer); + EntryBitfieldUpdate { + drop: flags == 1, + start, + length, + } + } +} /// Oplog Entry #[derive(Debug)] @@ -13,14 +137,85 @@ pub struct Entry { // } // TODO: This is a keyValueArray in JS pub(crate) user_data: Vec, + pub(crate) tree_nodes: Vec, + pub(crate) tree_upgrade: Option, + pub(crate) bitfield: Option, } impl CompactEncoding for State { - fn preencode(&mut self, value: &Entry) {} + fn preencode(&mut self, value: &Entry) { + self.end += 1; // flags + if value.user_data.len() > 0 { + self.preencode(&value.user_data); + } + if value.tree_nodes.len() > 0 { + self.preencode(&value.tree_nodes); + } + if let Some(tree_upgrade) = &value.tree_upgrade { + self.preencode(tree_upgrade); + } + if let Some(bitfield) = &value.bitfield { + self.preencode(bitfield); + } + } + + fn encode(&mut self, value: &Entry, buffer: &mut [u8]) { + let start = self.start; + self.start += 1; + let mut flags: u8 = 0; + if value.user_data.len() > 0 { + flags = flags | 1; + self.encode(&value.user_data, buffer); + } + if value.tree_nodes.len() > 0 { + flags = flags | 2; + self.encode(&value.tree_nodes, buffer); + } + if let Some(tree_upgrade) = &value.tree_upgrade { + flags = flags | 4; + self.encode(tree_upgrade, buffer); + } + if let Some(bitfield) = &value.bitfield { + flags = flags | 8; + self.encode(bitfield, buffer); + } - fn encode(&mut self, value: &Entry, buffer: &mut [u8]) {} + buffer[start] = flags; + } fn decode(&mut self, buffer: &[u8]) -> Entry { - Entry { user_data: vec![] } + let flags = self.decode_u8(buffer); + let user_data: Vec = if flags & 1 != 0 { + self.decode(buffer) + } else { + vec![] + }; + + let tree_nodes: Vec = if flags & 2 != 0 { + self.decode(buffer) + } else { + vec![] + }; + + let tree_upgrade: Option = if flags & 4 != 0 { + let value: EntryTreeUpgrade = self.decode(buffer); + Some(value) + } else { + None + }; + + let bitfield: Option = if flags & 4 != 0 { + let value: EntryBitfieldUpdate = self.decode(buffer); + Some(value) + } else { + None + }; + + Entry { + user_data, + tree_nodes, + tree_upgrade, + bitfield, + } } } From 913ec4474020a9e3ed2ab9f3bb44d7cdb245de9d Mon Sep 17 00:00:00 2001 From: Timo Tiuraniemi Date: Tue, 6 Sep 2022 09:57:11 +0300 Subject: [PATCH 035/157] The correct data is written --- src/core.rs | 18 ++++++++++++++---- src/data/mod.rs | 23 +++++++++++++++++++++++ src/lib.rs | 2 ++ src/storage_v10/mod.rs | 7 ++++++- src/tree/merkle_tree.rs | 8 ++++---- src/tree/merkle_tree_changeset.rs | 7 ++++--- 6 files changed, 53 insertions(+), 12 deletions(-) create mode 100644 src/data/mod.rs diff --git a/src/core.rs b/src/core.rs index e1a968fa..90edcc93 100644 --- a/src/core.rs +++ b/src/core.rs @@ -2,7 +2,7 @@ pub use crate::storage_v10::{PartialKeypair, Storage, Store}; -use crate::{crypto::generate_keypair, oplog::Oplog, sign, tree::MerkleTree}; +use crate::{crypto::generate_keypair, data::BlockStore, oplog::Oplog, sign, tree::MerkleTree}; use anyhow::Result; use random_access_storage::RandomAccess; use std::fmt::Debug; @@ -17,6 +17,7 @@ where pub(crate) storage: Storage, pub(crate) oplog: Oplog, pub(crate) tree: MerkleTree, + pub(crate) block_store: BlockStore, // /// Bitfield to keep track of which data we own. // pub(crate) bitfield: Bitfield, } @@ -63,15 +64,19 @@ where let slices = storage.read_slices(Store::Tree, slice_instructions).await?; let tree = MerkleTree::open(&oplog_open_outcome.header.tree, slices)?; + // Create block store instance + let block_store = BlockStore::default(); + Ok(Hypercore { key_pair, storage, oplog: oplog_open_outcome.oplog, tree, + block_store, }) } - /// Appends a given batch of bytes to the hypercore. + /// Appends a given batch of data blobs to the hypercore. pub async fn append_batch(&mut self, batch: Vec<&[u8]>) -> Result { let secret_key = match &self.key_pair.secret { Some(key) => key, @@ -79,10 +84,15 @@ where }; let mut changeset = self.tree.changeset(); - for data in batch { - changeset.append(data); + let mut batch_length: usize = 0; + for data in &batch { + batch_length += changeset.append(data); } let hash = changeset.hash_and_sign(&self.key_pair.public, &secret_key); + let slice = self + .block_store + .append_batch(batch, batch_length, self.tree.byte_length); + self.storage.flush_slice(Store::Data, slice).await?; Ok(AppendOutcome { length: 0, diff --git a/src/data/mod.rs b/src/data/mod.rs new file mode 100644 index 00000000..40acfe04 --- /dev/null +++ b/src/data/mod.rs @@ -0,0 +1,23 @@ +use crate::common::BufferSlice; + +/// Block store +#[derive(Debug, Default)] +pub struct BlockStore {} + +impl BlockStore { + pub fn append_batch( + &self, + batch: Vec<&[u8]>, + batch_length: usize, + byte_length: u64, + ) -> BufferSlice { + let mut buffer: Vec = Vec::with_capacity(batch_length); + for data in &batch { + buffer.extend_from_slice(data); + } + BufferSlice { + index: byte_length, + data: Some(buffer.into_boxed_slice()), + } + } +} diff --git a/src/lib.rs b/src/lib.rs index a5c15cbe..5077612e 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -48,6 +48,8 @@ mod common; #[cfg(feature = "v10")] mod core; mod crypto; +#[cfg(feature = "v10")] +mod data; mod event; #[cfg(not(feature = "v10"))] mod feed; diff --git a/src/storage_v10/mod.rs b/src/storage_v10/mod.rs index a856dfd8..068ae434 100644 --- a/src/storage_v10/mod.rs +++ b/src/storage_v10/mod.rs @@ -119,7 +119,12 @@ where Ok(slices.into_boxed_slice()) } - /// Flushes slices. Either writes directly to a random access storage or truncates the storage + /// Flush slice to storage. Convenience method to `flush_slices`. + pub async fn flush_slice(&mut self, store: Store, slice: BufferSlice) -> Result<()> { + self.flush_slices(store, [slice].into()).await + } + + /// Flush slices to storage. Either writes directly to a random access storage or truncates the storage /// to the length of given index. pub async fn flush_slices(&mut self, store: Store, slices: Box<[BufferSlice]>) -> Result<()> { let storage = self.get_random_access(store); diff --git a/src/tree/merkle_tree.rs b/src/tree/merkle_tree.rs index 61bb88f0..627c756a 100644 --- a/src/tree/merkle_tree.rs +++ b/src/tree/merkle_tree.rs @@ -14,10 +14,10 @@ use super::MerkleTreeChangeset; /// See https://github.com/hypercore-protocol/hypercore/blob/master/lib/merkle-tree.js #[derive(Debug)] pub struct MerkleTree { - roots: Vec, - length: u64, - byte_length: u64, - fork: u64, + pub(crate) roots: Vec, + pub(crate) length: u64, + pub(crate) byte_length: u64, + pub(crate) fork: u64, } const NODE_SIZE: u64 = 40; diff --git a/src/tree/merkle_tree_changeset.rs b/src/tree/merkle_tree_changeset.rs index 945edec0..09f78b9c 100644 --- a/src/tree/merkle_tree_changeset.rs +++ b/src/tree/merkle_tree_changeset.rs @@ -33,12 +33,13 @@ impl MerkleTreeChangeset { } } - pub fn append(&mut self, data: &[u8]) { - let len = data.len() as u64; + pub fn append(&mut self, data: &[u8]) -> usize { + let len = data.len(); let head = self.length * 2; let iter = flat_tree::Iterator::new(head); - let node = Node::new(head, Hash::data(data).as_bytes().to_vec(), len); + let node = Node::new(head, Hash::data(data).as_bytes().to_vec(), len as u64); self.append_root(node, iter); + len } pub fn append_root(&mut self, node: Node, iter: flat_tree::Iterator) { From 5a6d19d5db630cb11b38cce3e22711029e21aa42 Mon Sep 17 00:00:00 2001 From: Timo Tiuraniemi Date: Tue, 6 Sep 2022 11:20:33 +0300 Subject: [PATCH 036/157] Some part of entries are written to oplog --- src/core.rs | 25 ++++++++++++++--- src/data/mod.rs | 4 +-- src/oplog/mod.rs | 64 +++++++++++++++++++++++++++++++++++++----- src/storage_v10/mod.rs | 4 +-- tests/js_interop.rs | 2 +- 5 files changed, 83 insertions(+), 16 deletions(-) diff --git a/src/core.rs b/src/core.rs index 90edcc93..452cdbf4 100644 --- a/src/core.rs +++ b/src/core.rs @@ -2,7 +2,14 @@ pub use crate::storage_v10::{PartialKeypair, Storage, Store}; -use crate::{crypto::generate_keypair, data::BlockStore, oplog::Oplog, sign, tree::MerkleTree}; +use crate::{ + crypto::generate_keypair, + data::BlockStore, + oplog::{Entry, EntryBitfieldUpdate, Oplog}, + sign, + tree::MerkleTree, + Node, +}; use anyhow::Result; use random_access_storage::RandomAccess; use std::fmt::Debug; @@ -55,7 +62,7 @@ where let oplog_bytes = storage.read_all(Store::Oplog).await?; let oplog_open_outcome = Oplog::open(key_pair.clone(), oplog_bytes)?; storage - .flush_slices(Store::Oplog, oplog_open_outcome.slices_to_flush) + .flush_slices(Store::Oplog, &oplog_open_outcome.slices_to_flush) .await?; // Open/create tree @@ -77,7 +84,7 @@ where } /// Appends a given batch of data blobs to the hypercore. - pub async fn append_batch(&mut self, batch: Vec<&[u8]>) -> Result { + pub async fn append_batch(&mut self, batch: &[&[u8]]) -> Result { let secret_key = match &self.key_pair.secret { Some(key) => key, None => anyhow::bail!("No secret key, cannot append."), @@ -85,7 +92,7 @@ where let mut changeset = self.tree.changeset(); let mut batch_length: usize = 0; - for data in &batch { + for data in batch.iter() { batch_length += changeset.append(data); } let hash = changeset.hash_and_sign(&self.key_pair.public, &secret_key); @@ -94,6 +101,16 @@ where .append_batch(batch, batch_length, self.tree.byte_length); self.storage.flush_slice(Store::Data, slice).await?; + let tree_nodes: Vec = changeset.nodes; + let entry: Entry = Entry { + user_data: vec![], + tree_nodes, + tree_upgrade: None, + bitfield: None, + }; + let slices = self.oplog.append(entry, false)?; + self.storage.flush_slices(Store::Oplog, &slices).await?; + Ok(AppendOutcome { length: 0, byte_length: 0, diff --git a/src/data/mod.rs b/src/data/mod.rs index 40acfe04..2d353f31 100644 --- a/src/data/mod.rs +++ b/src/data/mod.rs @@ -7,12 +7,12 @@ pub struct BlockStore {} impl BlockStore { pub fn append_batch( &self, - batch: Vec<&[u8]>, + batch: &[&[u8]], batch_length: usize, byte_length: u64, ) -> BufferSlice { let mut buffer: Vec = Vec::with_capacity(batch_length); - for data in &batch { + for data in batch.iter() { buffer.extend_from_slice(data); } BufferSlice { diff --git a/src/oplog/mod.rs b/src/oplog/mod.rs index f27a28a3..8f6623fc 100644 --- a/src/oplog/mod.rs +++ b/src/oplog/mod.rs @@ -7,7 +7,7 @@ use std::convert::{TryFrom, TryInto}; mod entry; mod header; -pub use entry::Entry; +pub use entry::{Entry, EntryBitfieldUpdate, EntryTreeUpgrade}; pub use header::{Header, HeaderTree}; /// Oplog. @@ -18,7 +18,8 @@ pub use header::{Header, HeaderTree}; #[derive(Debug)] pub struct Oplog { header_bits: [bool; 2], - entries_len: u64, + entries_length: u64, + entries_byte_length: u64, } /// Oplog @@ -48,7 +49,6 @@ const INITIAL_HEADER_BITS: [bool; 2] = [true, false]; impl Oplog { /// Opens an existing Oplog from existing byte buffer or creates a new one. - #[allow(dead_code)] pub fn open(key_pair: PartialKeypair, existing: Box<[u8]>) -> Result { // First read and validate both headers stored in the existing oplog let h1_outcome = Self::validate_leader(OplogSlot::FirstHeader as usize, &existing)?; @@ -74,7 +74,8 @@ impl Oplog { }; let oplog = Oplog { header_bits, - entries_len: 0, + entries_length: 0, + entries_byte_length: 0, }; Ok(OplogOpenOutcome { oplog, @@ -87,7 +88,8 @@ impl Oplog { let header_bits: [bool; 2] = [!h2_outcome.header_bit, h2_outcome.header_bit]; let oplog = Oplog { header_bits, - entries_len: 0, + entries_length: 0, + entries_byte_length: 0, }; Ok(OplogOpenOutcome { oplog, @@ -100,10 +102,53 @@ impl Oplog { } } + /// Appends a entry to the Oplog. + pub fn append(&mut self, entry: Entry, atomic: bool) -> Result> { + self.append_batch(&[entry], atomic) + } + + /// Appends a batch of entries to the Oplog. + pub fn append_batch(&mut self, batch: &[Entry], atomic: bool) -> Result> { + let len = batch.len(); + let header_bit = self.get_current_header_bit(); + // Leave room for leaders + let mut state = State::new_with_start_and_end(0, len * 8); + + for entry in batch.iter() { + state.preencode(entry); + } + + let mut buffer = state.create_buffer(); + for i in 0..len { + let entry = &batch[i]; + state.start += 8; + let start = state.start; + let partial_bit: bool = atomic && i < len - 1; + state.encode(entry, &mut buffer); + Self::prepend_leader( + state.start - start, + header_bit, + partial_bit, + &mut state, + &mut buffer, + ); + } + + self.entries_length += len as u64; + self.entries_byte_length += buffer.len() as u64; + + Ok(vec![BufferSlice { + index: OplogSlot::Entries as u64 + self.entries_byte_length, + data: Some(buffer), + }] + .into_boxed_slice()) + } + fn new(key_pair: PartialKeypair) -> OplogOpenOutcome { let oplog = Oplog { header_bits: INITIAL_HEADER_BITS, - entries_len: 0, + entries_length: 0, + entries_byte_length: 0, }; // The first 8 bytes will be filled with `prepend_leader`. @@ -135,7 +180,7 @@ impl Oplog { // The oplog is always truncated to the minimum byte size, which is right after // the all of the entries in the oplog finish. - let truncate_index = OplogSlot::Entries as u64 + oplog.entries_len; + let truncate_index = OplogSlot::Entries as u64 + oplog.entries_byte_length; OplogOpenOutcome { oplog, header, @@ -216,6 +261,11 @@ impl Oplog { })) } + /// Gets the current header bit + fn get_current_header_bit(&self) -> bool { + self.header_bits[0] != self.header_bits[1] + } + /// Based on given header_bits, determines if saving the header should be done to the first /// header slot or the second header slot and the bit that it should get. fn get_next_header_oplog_slot_and_bit_value(header_bits: &[bool; 2]) -> (OplogSlot, bool) { diff --git a/src/storage_v10/mod.rs b/src/storage_v10/mod.rs index 068ae434..4780a9f5 100644 --- a/src/storage_v10/mod.rs +++ b/src/storage_v10/mod.rs @@ -121,12 +121,12 @@ where /// Flush slice to storage. Convenience method to `flush_slices`. pub async fn flush_slice(&mut self, store: Store, slice: BufferSlice) -> Result<()> { - self.flush_slices(store, [slice].into()).await + self.flush_slices(store, &[slice]).await } /// Flush slices to storage. Either writes directly to a random access storage or truncates the storage /// to the length of given index. - pub async fn flush_slices(&mut self, store: Store, slices: Box<[BufferSlice]>) -> Result<()> { + pub async fn flush_slices(&mut self, store: Store, slices: &[BufferSlice]) -> Result<()> { let storage = self.get_random_access(store); for slice in slices.iter() { match &slice.data { diff --git a/tests/js_interop.rs b/tests/js_interop.rs index c01dac6f..20606efd 100644 --- a/tests/js_interop.rs +++ b/tests/js_interop.rs @@ -62,7 +62,7 @@ async fn step_2_append_hello_world(work_dir: &str) -> Result<()> { let key_pair = get_test_key_pair(); let storage = Storage::new_disk(&path, false).await?; let mut hypercore = Hypercore::new_with_key_pair(storage, key_pair).await?; - let _append_outcome = hypercore.append_batch(vec![b"Hello", b"World"]).await?; + let _append_outcome = hypercore.append_batch(&[b"Hello", b"World"]).await?; Ok(()) } From 062e0480cd0196037204cc2e16cedab5abe43bd9 Mon Sep 17 00:00:00 2001 From: Timo Tiuraniemi Date: Tue, 6 Sep 2022 11:45:17 +0300 Subject: [PATCH 037/157] Oplog should be roughly correct --- src/core.rs | 25 +++++++++++++++++++++---- src/tree/merkle_tree_changeset.rs | 11 ++++++----- tests/js_interop.rs | 18 +++++++++--------- 3 files changed, 36 insertions(+), 18 deletions(-) diff --git a/src/core.rs b/src/core.rs index 452cdbf4..5251ce50 100644 --- a/src/core.rs +++ b/src/core.rs @@ -5,7 +5,7 @@ pub use crate::storage_v10::{PartialKeypair, Storage, Store}; use crate::{ crypto::generate_keypair, data::BlockStore, - oplog::{Entry, EntryBitfieldUpdate, Oplog}, + oplog::{Entry, EntryBitfieldUpdate, EntryTreeUpgrade, Oplog}, sign, tree::MerkleTree, Node, @@ -90,27 +90,44 @@ where None => anyhow::bail!("No secret key, cannot append."), }; + // Create a changeset for the tree let mut changeset = self.tree.changeset(); let mut batch_length: usize = 0; for data in batch.iter() { batch_length += changeset.append(data); } - let hash = changeset.hash_and_sign(&self.key_pair.public, &secret_key); + changeset.hash_and_sign(&self.key_pair.public, &secret_key); + + // Write the received data to the block store let slice = self .block_store .append_batch(batch, batch_length, self.tree.byte_length); self.storage.flush_slice(Store::Data, slice).await?; + // Create an Oplog entry from the changeset let tree_nodes: Vec = changeset.nodes; let entry: Entry = Entry { user_data: vec![], tree_nodes, - tree_upgrade: None, - bitfield: None, + tree_upgrade: Some(EntryTreeUpgrade { + fork: changeset.fork, + ancestors: changeset.ancestors, + length: changeset.length, + signature: changeset.hash_and_signature.unwrap().1.to_bytes().into(), + }), + bitfield: Some(EntryBitfieldUpdate { + drop: false, + start: changeset.ancestors, + length: batch.len() as u64, + }), }; + + // Write the oplog entry to the oplog store let slices = self.oplog.append(entry, false)?; self.storage.flush_slices(Store::Oplog, &slices).await?; + // TODO: write bitfield + Ok(AppendOutcome { length: 0, byte_length: 0, diff --git a/src/tree/merkle_tree_changeset.rs b/src/tree/merkle_tree_changeset.rs index 09f78b9c..9022d422 100644 --- a/src/tree/merkle_tree_changeset.rs +++ b/src/tree/merkle_tree_changeset.rs @@ -14,22 +14,24 @@ use crate::{ #[derive(Debug)] pub struct MerkleTreeChangeset { pub(crate) length: u64, + pub(crate) ancestors: u64, pub(crate) byte_length: u64, pub(crate) fork: u64, pub(crate) roots: Vec, pub(crate) nodes: Vec, - pub(crate) signature: Option, + pub(crate) hash_and_signature: Option<(Box<[u8]>, Signature)>, } impl MerkleTreeChangeset { pub fn new(length: u64, byte_length: u64, fork: u64, roots: Vec) -> MerkleTreeChangeset { Self { length, + ancestors: length, byte_length, fork, roots, nodes: vec![], - signature: None, + hash_and_signature: None, } } @@ -70,12 +72,11 @@ impl MerkleTreeChangeset { } /// Hashes and signs the changeset - pub fn hash_and_sign(&mut self, public_key: &PublicKey, secret_key: &SecretKey) -> Box<[u8]> { + pub fn hash_and_sign(&mut self, public_key: &PublicKey, secret_key: &SecretKey) { let hash = self.hash(); let signable = signable_tree(&hash, self.length, self.fork); let signature = sign(&public_key, &secret_key, &signable); - self.signature = Some(signature); - hash + self.hash_and_signature = Some((hash, signature)); } /// Calculates a hash of the current set of roots diff --git a/tests/js_interop.rs b/tests/js_interop.rs index 20606efd..8d00cfed 100644 --- a/tests/js_interop.rs +++ b/tests/js_interop.rs @@ -25,11 +25,11 @@ fn init() { async fn js_interop_js_first() -> Result<()> { init(); let work_dir = prepare_test_set(TEST_SET_JS_FIRST); - assert_eq!(get_step_0_hash(), create_hypercore_hash(&work_dir)); + assert_eq!(step_0_hash(), create_hypercore_hash(&work_dir)); js_run_step(1, TEST_SET_JS_FIRST); - assert_eq!(get_step_1_hash(), create_hypercore_hash(&work_dir)); + assert_eq!(step_1_hash(), create_hypercore_hash(&work_dir)); step_2_append_hello_world(&work_dir).await?; - assert_eq!(get_step_2_hash(), create_hypercore_hash(&work_dir)); + assert_eq!(step_2_hash(), create_hypercore_hash(&work_dir)); Ok(()) } @@ -39,11 +39,11 @@ async fn js_interop_js_first() -> Result<()> { async fn js_interop_rs_first() -> Result<()> { init(); let work_dir = prepare_test_set(TEST_SET_RS_FIRST); - assert_eq!(get_step_0_hash(), create_hypercore_hash(&work_dir)); + assert_eq!(step_0_hash(), create_hypercore_hash(&work_dir)); step_1_create(&work_dir).await?; - assert_eq!(get_step_1_hash(), create_hypercore_hash(&work_dir)); + assert_eq!(step_1_hash(), create_hypercore_hash(&work_dir)); js_run_step(2, TEST_SET_RS_FIRST); - assert_eq!(get_step_2_hash(), create_hypercore_hash(&work_dir)); + assert_eq!(step_2_hash(), create_hypercore_hash(&work_dir)); Ok(()) } @@ -66,7 +66,7 @@ async fn step_2_append_hello_world(work_dir: &str) -> Result<()> { Ok(()) } -fn get_step_0_hash() -> common::HypercoreHash { +fn step_0_hash() -> common::HypercoreHash { common::HypercoreHash { bitfield: None, data: None, @@ -75,7 +75,7 @@ fn get_step_0_hash() -> common::HypercoreHash { } } -fn get_step_1_hash() -> common::HypercoreHash { +fn step_1_hash() -> common::HypercoreHash { common::HypercoreHash { bitfield: None, data: None, @@ -84,7 +84,7 @@ fn get_step_1_hash() -> common::HypercoreHash { } } -fn get_step_2_hash() -> common::HypercoreHash { +fn step_2_hash() -> common::HypercoreHash { common::HypercoreHash { bitfield: Some("0E2E1FF956A39192CBB68D2212288FE75B32733AB0C442B9F0471E254A0382A2".into()), data: Some("872E4E50CE9990D8B041330C47C9DDD11BEC6B503AE9386A99DA8584E9BB12C4".into()), From b5e2ae642e8bcf780ba13b193fa081b6337b3a82 Mon Sep 17 00:00:00 2001 From: Timo Tiuraniemi Date: Tue, 6 Sep 2022 13:18:36 +0300 Subject: [PATCH 038/157] Creates an identical oplog to JS --- src/core.rs | 22 ++-------------- src/oplog/mod.rs | 42 ++++++++++++++++++++++++----- src/tree/merkle_tree_changeset.rs | 44 ++++++++++++++++--------------- 3 files changed, 61 insertions(+), 47 deletions(-) diff --git a/src/core.rs b/src/core.rs index 5251ce50..5352f91b 100644 --- a/src/core.rs +++ b/src/core.rs @@ -104,26 +104,8 @@ where .append_batch(batch, batch_length, self.tree.byte_length); self.storage.flush_slice(Store::Data, slice).await?; - // Create an Oplog entry from the changeset - let tree_nodes: Vec = changeset.nodes; - let entry: Entry = Entry { - user_data: vec![], - tree_nodes, - tree_upgrade: Some(EntryTreeUpgrade { - fork: changeset.fork, - ancestors: changeset.ancestors, - length: changeset.length, - signature: changeset.hash_and_signature.unwrap().1.to_bytes().into(), - }), - bitfield: Some(EntryBitfieldUpdate { - drop: false, - start: changeset.ancestors, - length: batch.len() as u64, - }), - }; - - // Write the oplog entry to the oplog store - let slices = self.oplog.append(entry, false)?; + // Append the changeset to the Oplog + let slices = self.oplog.append_changeset(&changeset, false)?; self.storage.flush_slices(Store::Oplog, &slices).await?; // TODO: write bitfield diff --git a/src/oplog/mod.rs b/src/oplog/mod.rs index 8f6623fc..d9f21a31 100644 --- a/src/oplog/mod.rs +++ b/src/oplog/mod.rs @@ -1,6 +1,7 @@ use crate::common::BufferSlice; use crate::compact_encoding::{CompactEncoding, State}; -use crate::PartialKeypair; +use crate::tree::MerkleTreeChangeset; +use crate::{Node, PartialKeypair}; use anyhow::{anyhow, Result}; use std::convert::{TryFrom, TryInto}; @@ -102,13 +103,41 @@ impl Oplog { } } - /// Appends a entry to the Oplog. - pub fn append(&mut self, entry: Entry, atomic: bool) -> Result> { - self.append_batch(&[entry], atomic) + /// Appends a changeset to the Oplog. + pub fn append_changeset( + &mut self, + changeset: &MerkleTreeChangeset, + atomic: bool, + ) -> Result> { + // TODO: Unsure if clone() is absolutely needed here or not. + let tree_nodes: Vec = changeset.nodes.clone(); + let entry: Entry = Entry { + user_data: vec![], + tree_nodes, + tree_upgrade: Some(EntryTreeUpgrade { + fork: changeset.fork, + ancestors: changeset.ancestors, + length: changeset.length, + signature: changeset + .hash_and_signature + .as_ref() + .unwrap() + .1 + .to_bytes() + .into(), + }), + bitfield: Some(EntryBitfieldUpdate { + drop: false, + start: changeset.ancestors, + length: changeset.batch_length, + }), + }; + + self.append_entries(&[entry], atomic) } /// Appends a batch of entries to the Oplog. - pub fn append_batch(&mut self, batch: &[Entry], atomic: bool) -> Result> { + fn append_entries(&mut self, batch: &[Entry], atomic: bool) -> Result> { let len = batch.len(); let header_bit = self.get_current_header_bit(); // Leave room for leaders @@ -134,11 +163,12 @@ impl Oplog { ); } + let index = OplogSlot::Entries as u64 + self.entries_byte_length; self.entries_length += len as u64; self.entries_byte_length += buffer.len() as u64; Ok(vec![BufferSlice { - index: OplogSlot::Entries as u64 + self.entries_byte_length, + index, data: Some(buffer), }] .into_boxed_slice()) diff --git a/src/tree/merkle_tree_changeset.rs b/src/tree/merkle_tree_changeset.rs index 9022d422..161caa41 100644 --- a/src/tree/merkle_tree_changeset.rs +++ b/src/tree/merkle_tree_changeset.rs @@ -16,6 +16,7 @@ pub struct MerkleTreeChangeset { pub(crate) length: u64, pub(crate) ancestors: u64, pub(crate) byte_length: u64, + pub(crate) batch_length: u64, pub(crate) fork: u64, pub(crate) roots: Vec, pub(crate) nodes: Vec, @@ -28,6 +29,7 @@ impl MerkleTreeChangeset { length, ancestors: length, byte_length, + batch_length: 0, fork, roots, nodes: vec![], @@ -38,37 +40,37 @@ impl MerkleTreeChangeset { pub fn append(&mut self, data: &[u8]) -> usize { let len = data.len(); let head = self.length * 2; - let iter = flat_tree::Iterator::new(head); + let mut iter = flat_tree::Iterator::new(head); let node = Node::new(head, Hash::data(data).as_bytes().to_vec(), len as u64); - self.append_root(node, iter); + self.append_root(node, &mut iter); + self.batch_length += 1; len } - pub fn append_root(&mut self, node: Node, iter: flat_tree::Iterator) { + pub fn append_root(&mut self, node: Node, iter: &mut flat_tree::Iterator) { self.length += iter.factor() / 2; self.byte_length += node.length; self.roots.push(node.clone()); self.nodes.push(node); - // TODO: Javascript has this - // - // while (this.roots.length > 1) { - // const a = this.roots[this.roots.length - 1] - // const b = this.roots[this.roots.length - 2] + while self.roots.len() > 1 { + let a = &self.roots[self.roots.len() - 1]; + let b = &self.roots[self.roots.len() - 2]; + if iter.sibling() != b.index { + iter.sibling(); // unset so it always points to last root + break; + } - // // TODO: just have a peek sibling instead? (pretty sure it's always the left sib as well) - // if (ite.sibling() !== b.index) { - // ite.sibling() // unset so it always points to last root - // break - // } - - // const node = parentNode(this.tree.crypto, ite.parent(), a, b) - // this.nodes.push(node) - // this.roots.pop() - // this.roots.pop() - // this.roots.push(node) - // } - // } + let node = Node::new( + iter.parent(), + Hash::parent(&a, &b).as_bytes().into(), + a.length + b.length, + ); + let _ = &self.nodes.push(node.clone()); + let _ = &self.roots.pop(); + let _ = &self.roots.pop(); + let _ = &self.roots.push(node); + } } /// Hashes and signs the changeset From 5c72fa443447d50813b5225eeecb5d7fd64f6db7 Mon Sep 17 00:00:00 2001 From: Timo Tiuraniemi Date: Tue, 6 Sep 2022 15:02:29 +0300 Subject: [PATCH 039/157] Almost the right oplog --- src/core.rs | 44 ++++++++++++++-- src/oplog/header.rs | 20 ++++---- src/oplog/mod.rs | 119 +++++++++++++++++++++++++++++++++----------- 3 files changed, 138 insertions(+), 45 deletions(-) diff --git a/src/core.rs b/src/core.rs index 5352f91b..216f76f5 100644 --- a/src/core.rs +++ b/src/core.rs @@ -5,10 +5,8 @@ pub use crate::storage_v10::{PartialKeypair, Storage, Store}; use crate::{ crypto::generate_keypair, data::BlockStore, - oplog::{Entry, EntryBitfieldUpdate, EntryTreeUpgrade, Oplog}, - sign, + oplog::{Header, Oplog, MAX_OPLOG_ENTRIES_BYTE_SIZE}, tree::MerkleTree, - Node, }; use anyhow::Result; use random_access_storage::RandomAccess; @@ -27,6 +25,8 @@ where pub(crate) block_store: BlockStore, // /// Bitfield to keep track of which data we own. // pub(crate) bitfield: Bitfield, + skip_flush_count: u8, // autoFlush in Javascript + header: Header, } /// Response from append, matches that of the Javascript result @@ -80,6 +80,8 @@ where oplog: oplog_open_outcome.oplog, tree, block_store, + skip_flush_count: 0, + header: oplog_open_outcome.header, }) } @@ -105,14 +107,46 @@ where self.storage.flush_slice(Store::Data, slice).await?; // Append the changeset to the Oplog - let slices = self.oplog.append_changeset(&changeset, false)?; - self.storage.flush_slices(Store::Oplog, &slices).await?; + let outcome = self.oplog.append_changeset(&changeset, false, &self.header); + self.storage + .flush_slices(Store::Oplog, &outcome.slices_to_flush) + .await?; + self.header = outcome.header; // TODO: write bitfield + // + if self.should_flush_bitfield_and_tree_and_oplog() { + self.flush_bitfield_and_tree_and_oplog().await?; + } Ok(AppendOutcome { length: 0, byte_length: 0, }) } + + fn should_flush_bitfield_and_tree_and_oplog(&mut self) -> bool { + if self.skip_flush_count == 0 + || self.oplog.entries_byte_length >= MAX_OPLOG_ENTRIES_BYTE_SIZE + { + self.skip_flush_count = 4; + true + } else { + self.skip_flush_count -= 1; + false + } + } + + async fn flush_bitfield_and_tree_and_oplog(&mut self) -> Result<()> { + // TODO: + // let slices = self.bitfield.flush(); + // self.storage.flush_slices(Store::Bitfield, &slices).await?; + // let slices = self.tree.flush(); + // self.storage.flush_slices(Store::Tree, &slices).await?; + let slices_to_flush = self.oplog.flush(&self.header); + self.storage + .flush_slices(Store::Oplog, &slices_to_flush) + .await?; + Ok(()) + } } diff --git a/src/oplog/header.rs b/src/oplog/header.rs index 6bd00961..b37deb3d 100644 --- a/src/oplog/header.rs +++ b/src/oplog/header.rs @@ -3,7 +3,7 @@ use crate::crypto::{PublicKey, SecretKey}; use crate::PartialKeypair; /// Oplog header. -#[derive(Debug)] +#[derive(Debug, Clone)] pub struct Header { pub(crate) types: HeaderTypes, // TODO: This is a keyValueArray in JS @@ -16,8 +16,8 @@ pub struct Header { impl Header { /// Creates a new Header from given key pair - pub fn new(key_pair: PartialKeypair) -> Header { - Header { + pub fn new(key_pair: PartialKeypair) -> Self { + Self { types: HeaderTypes::new(), user_data: vec![], tree: HeaderTree::new(), @@ -46,15 +46,15 @@ impl Header { } /// Oplog header types -#[derive(Debug, PartialEq)] +#[derive(Debug, PartialEq, Clone)] pub struct HeaderTypes { pub(crate) tree: String, pub(crate) bitfield: String, pub(crate) signer: String, } impl HeaderTypes { - pub fn new() -> HeaderTypes { - HeaderTypes { + pub fn new() -> Self { + Self { tree: "blake2b".to_string(), bitfield: "raw".to_string(), signer: "ed25519".to_string(), @@ -88,7 +88,7 @@ impl CompactEncoding for State { } /// Oplog header tree -#[derive(Debug, PartialEq)] +#[derive(Debug, PartialEq, Clone)] pub struct HeaderTree { pub(crate) fork: u64, pub(crate) length: u64, @@ -97,8 +97,8 @@ pub struct HeaderTree { } impl HeaderTree { - pub fn new() -> HeaderTree { - HeaderTree { + pub fn new() -> Self { + Self { fork: 0, length: 0, root_hash: Box::new([]), @@ -187,7 +187,7 @@ impl CompactEncoding for State { } /// Oplog header hints -#[derive(Debug)] +#[derive(Debug, Clone)] pub struct HeaderHints { pub(crate) reorgs: Vec, } diff --git a/src/oplog/mod.rs b/src/oplog/mod.rs index d9f21a31..f21f55c6 100644 --- a/src/oplog/mod.rs +++ b/src/oplog/mod.rs @@ -11,6 +11,8 @@ mod header; pub use entry::{Entry, EntryBitfieldUpdate, EntryTreeUpgrade}; pub use header::{Header, HeaderTree}; +pub const MAX_OPLOG_ENTRIES_BYTE_SIZE: u64 = 65536; + /// Oplog. /// /// There are two memory areas for an `Header` in `RandomAccessStorage`: one is the current @@ -19,11 +21,18 @@ pub use header::{Header, HeaderTree}; #[derive(Debug)] pub struct Oplog { header_bits: [bool; 2], - entries_length: u64, - entries_byte_length: u64, + pub(crate) entries_length: u64, + pub(crate) entries_byte_length: u64, } -/// Oplog +/// Oplog create header outcome +#[derive(Debug)] +pub struct OplogCreateHeaderOutcome { + pub header: Header, + pub slices_to_flush: Box<[BufferSlice]>, +} + +/// Oplog open outcome #[derive(Debug)] pub struct OplogOpenOutcome { pub oplog: Oplog, @@ -31,6 +40,16 @@ pub struct OplogOpenOutcome { pub slices_to_flush: Box<[BufferSlice]>, } +impl OplogOpenOutcome { + pub fn new(oplog: Oplog, create_header_outcome: OplogCreateHeaderOutcome) -> Self { + Self { + oplog, + header: create_header_outcome.header, + slices_to_flush: create_header_outcome.slices_to_flush, + } + } +} + enum OplogSlot { FirstHeader = 0, SecondHeader = 4096, @@ -108,9 +127,14 @@ impl Oplog { &mut self, changeset: &MerkleTreeChangeset, atomic: bool, - ) -> Result> { - // TODO: Unsure if clone() is absolutely needed here or not. + header: &Header, + ) -> OplogCreateHeaderOutcome { let tree_nodes: Vec = changeset.nodes.clone(); + let (hash, signature) = changeset + .hash_and_signature + .as_ref() + .expect("Changeset must be signed before appended"); + let signature: Box<[u8]> = signature.to_bytes().into(); let entry: Entry = Entry { user_data: vec![], tree_nodes, @@ -118,13 +142,7 @@ impl Oplog { fork: changeset.fork, ancestors: changeset.ancestors, length: changeset.length, - signature: changeset - .hash_and_signature - .as_ref() - .unwrap() - .1 - .to_bytes() - .into(), + signature: signature.clone(), }), bitfield: Some(EntryBitfieldUpdate { drop: false, @@ -133,11 +151,28 @@ impl Oplog { }), }; - self.append_entries(&[entry], atomic) + let mut header: Header = header.clone(); + header.tree.length = changeset.batch_length; + header.tree.root_hash = hash.clone(); + header.tree.signature = signature; + + OplogCreateHeaderOutcome { + header, + slices_to_flush: self.append_entries(&[entry], atomic), + } + } + + /// Flushes pending changes, returns buffer slices to write to storage. + pub fn flush(&mut self, header: &Header) -> Box<[BufferSlice]> { + let (new_header_bits, slices_to_flush) = Self::insert_header(header, 0, self.header_bits); + self.entries_byte_length = 0; + self.entries_length = 0; + self.header_bits = new_header_bits; + slices_to_flush } /// Appends a batch of entries to the Oplog. - fn append_entries(&mut self, batch: &[Entry], atomic: bool) -> Result> { + fn append_entries(&mut self, batch: &[Entry], atomic: bool) -> Box<[BufferSlice]> { let len = batch.len(); let header_bit = self.get_current_header_bit(); // Leave room for leaders @@ -167,37 +202,62 @@ impl Oplog { self.entries_length += len as u64; self.entries_byte_length += buffer.len() as u64; - Ok(vec![BufferSlice { + vec![BufferSlice { index, data: Some(buffer), }] - .into_boxed_slice()) + .into_boxed_slice() } fn new(key_pair: PartialKeypair) -> OplogOpenOutcome { + let entries_length: u64 = 0; + let entries_byte_length: u64 = 0; + let header = Header::new(key_pair); + let (header_bits, slices_to_flush) = + Self::insert_header(&header, entries_byte_length, INITIAL_HEADER_BITS); let oplog = Oplog { - header_bits: INITIAL_HEADER_BITS, - entries_length: 0, - entries_byte_length: 0, + header_bits, + entries_length, + entries_byte_length, }; + OplogOpenOutcome::new( + oplog, + OplogCreateHeaderOutcome { + header, + slices_to_flush, + }, + ) + } + fn insert_header( + header: &Header, + entries_byte_length: u64, + current_header_bits: [bool; 2], + ) -> ([bool; 2], Box<[BufferSlice]>) { // The first 8 bytes will be filled with `prepend_leader`. let data_start_index: usize = 8; let mut state = State::new_with_start_and_end(data_start_index, data_start_index); // Get the right slot and header bit let (oplog_slot, header_bit) = - Oplog::get_next_header_oplog_slot_and_bit_value(&oplog.header_bits); + Oplog::get_next_header_oplog_slot_and_bit_value(¤t_header_bits); + let mut new_header_bits = current_header_bits.clone(); + match oplog_slot { + OplogSlot::FirstHeader => new_header_bits[0] = header_bit, + OplogSlot::SecondHeader => new_header_bits[1] = header_bit, + _ => { + panic!("Invalid oplog slot"); + } + } - // Preencode a new header - let header = Header::new(key_pair); - state.preencode(&header); + // Preencode the new header + state.preencode(header); // Create a buffer for the needed data let mut buffer = state.create_buffer(); // Encode the header - state.encode(&header, &mut buffer); + state.encode(header, &mut buffer); // Finally prepend the buffer's 8 first bytes with a CRC, len and right bits Self::prepend_leader( @@ -210,11 +270,10 @@ impl Oplog { // The oplog is always truncated to the minimum byte size, which is right after // the all of the entries in the oplog finish. - let truncate_index = OplogSlot::Entries as u64 + oplog.entries_byte_length; - OplogOpenOutcome { - oplog, - header, - slices_to_flush: vec![ + let truncate_index = OplogSlot::Entries as u64 + entries_byte_length; + ( + new_header_bits, + vec![ BufferSlice { index: oplog_slot as u64, data: Some(buffer), @@ -225,7 +284,7 @@ impl Oplog { }, ] .into_boxed_slice(), - } + ) } /// Prepends given `State` with 4 bytes of CRC followed by 4 bytes containing length of From 2162ffaf858b9251aa249a9c83c77ecbf3f15e28 Mon Sep 17 00:00:00 2001 From: Timo Tiuraniemi Date: Tue, 6 Sep 2022 16:25:26 +0300 Subject: [PATCH 040/157] Added fixed bitfield --- src/bitfield_v10/dynamic.rs | 3 ++ src/bitfield_v10/fixed.rs | 78 +++++++++++++++++++++++++++++++++++++ src/bitfield_v10/mod.rs | 4 ++ src/core.rs | 2 +- src/lib.rs | 4 ++ tests/bitfield.rs | 9 +++++ 6 files changed, 99 insertions(+), 1 deletion(-) create mode 100644 src/bitfield_v10/dynamic.rs create mode 100644 src/bitfield_v10/fixed.rs create mode 100644 src/bitfield_v10/mod.rs diff --git a/src/bitfield_v10/dynamic.rs b/src/bitfield_v10/dynamic.rs new file mode 100644 index 00000000..f48e2b09 --- /dev/null +++ b/src/bitfield_v10/dynamic.rs @@ -0,0 +1,3 @@ +/// Dynamic sized bitfield, uses a collection of `FixeddBitfield` elements. +#[derive(Debug)] +pub struct DynamicBitfield {} diff --git a/src/bitfield_v10/fixed.rs b/src/bitfield_v10/fixed.rs new file mode 100644 index 00000000..70e04373 --- /dev/null +++ b/src/bitfield_v10/fixed.rs @@ -0,0 +1,78 @@ +const FIXED_BITFIELD_LENGTH: usize = 1024; +// u32 has 4 bytes and a byte has 8 bits +const FIXED_BITFIELD_BITS_PER_ELEM: u64 = 4 * 8; +use std::convert::TryInto; + +/// Fixed size bitfield +/// see: +/// https://github.com/holepunchto/bits-to-bytes/blob/main/index.js +/// for implementations. +#[derive(Debug)] +pub struct FixedBitfield { + pub(crate) parent_index: u64, + bitfield: [u32; FIXED_BITFIELD_LENGTH], +} + +impl FixedBitfield { + pub fn new(parent_index: u64) -> Self { + Self { + parent_index, + bitfield: [0; FIXED_BITFIELD_LENGTH], + } + } + + pub fn get(&self, index: u64) -> bool { + let n = FIXED_BITFIELD_BITS_PER_ELEM; + let offset = index & (n - 1); + let i: usize = ((index - offset) / n) + .try_into() + .expect("Could not fit 64 bit integer to usize on this architecture"); + self.bitfield[i] & (1 << offset) != 0 + } + + pub fn set(&mut self, index: u64, value: bool) -> bool { + let n = FIXED_BITFIELD_BITS_PER_ELEM; + let offset = index & (n - 1); + let i: usize = ((index - offset) / n) + .try_into() + .expect("Could not fit 64 bit integer to usize on this architecture"); + let mask = 1 << offset; + + if value { + if (self.bitfield[i] & mask) != 0 { + return false; + } + } else { + if (self.bitfield[i] & mask) == 0 { + return false; + } + } + self.bitfield[i] ^= mask; + true + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn bitfield_fixed_get_and_set() { + let mut bitfield = FixedBitfield::new(0); + assert_eq!(bitfield.get(0), false); + bitfield.set(0, true); + assert_eq!(bitfield.get(0), true); + + assert_eq!(bitfield.get(31), false); + bitfield.set(31, true); + assert_eq!(bitfield.get(31), true); + + assert_eq!(bitfield.get(32), false); + bitfield.set(32, true); + assert_eq!(bitfield.get(32), true); + + assert_eq!(bitfield.get(32767), false); + bitfield.set(32767, true); + assert_eq!(bitfield.get(32767), true); + } +} diff --git a/src/bitfield_v10/mod.rs b/src/bitfield_v10/mod.rs new file mode 100644 index 00000000..d15d70d5 --- /dev/null +++ b/src/bitfield_v10/mod.rs @@ -0,0 +1,4 @@ +mod dynamic; +mod fixed; + +pub use dynamic::DynamicBitfield as Bitfield; diff --git a/src/core.rs b/src/core.rs index 216f76f5..bd9a5448 100644 --- a/src/core.rs +++ b/src/core.rs @@ -113,7 +113,7 @@ where .await?; self.header = outcome.header; - // TODO: write bitfield + // TODO: write bitfield and contiguous length // if self.should_flush_bitfield_and_tree_and_oplog() { self.flush_bitfield_and_tree_and_oplog().await?; diff --git a/src/lib.rs b/src/lib.rs index 5077612e..5d1f67d2 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -39,11 +39,15 @@ //! [Dat]: https://github.com/datrs //! [Feed]: crate::feed::Feed +#[cfg(not(feature = "v10"))] pub mod bitfield; pub mod compact_encoding; pub mod prelude; mod audit; + +#[cfg(feature = "v10")] +mod bitfield_v10; mod common; #[cfg(feature = "v10")] mod core; diff --git a/tests/bitfield.rs b/tests/bitfield.rs index 566ac26e..6225a3d5 100644 --- a/tests/bitfield.rs +++ b/tests/bitfield.rs @@ -1,9 +1,12 @@ use rand; +#[cfg(not(feature = "v10"))] use hypercore::bitfield::{Bitfield, Change::*}; +#[cfg(not(feature = "v10"))] use rand::Rng; #[test] +#[cfg(not(feature = "v10"))] fn set_and_get() { let (mut b, _) = Bitfield::new(); @@ -19,6 +22,7 @@ fn set_and_get() { } #[test] +#[cfg(not(feature = "v10"))] fn set_and_get_tree() { let (mut b, mut tree) = Bitfield::new(); @@ -39,6 +43,7 @@ fn set_and_get_tree() { } #[test] +#[cfg(not(feature = "v10"))] fn set_and_index() { let (mut b, _) = Bitfield::new(); @@ -97,6 +102,7 @@ fn set_and_index() { } #[test] +#[cfg(not(feature = "v10"))] fn set_and_index_random() { let (mut b, _) = Bitfield::new(); @@ -136,6 +142,7 @@ fn set_and_index_random() { } #[test] +#[cfg(not(feature = "v10"))] fn get_total_positive_bits() { let (mut b, _) = Bitfield::new(); @@ -154,6 +161,7 @@ fn get_total_positive_bits() { } #[test] +#[cfg(not(feature = "v10"))] fn bitfield_dedup() { let (mut b, mut tree) = Bitfield::new(); @@ -173,6 +181,7 @@ fn bitfield_dedup() { } #[test] +#[cfg(not(feature = "v10"))] fn bitfield_compress() { let (mut b, _) = Bitfield::new(); assert_eq!(b.compress(0, 0).unwrap(), vec![0]); From 2285eef06ec22b9053081eec77653f7d4b3df77b Mon Sep 17 00:00:00 2001 From: Timo Tiuraniemi Date: Wed, 7 Sep 2022 10:27:12 +0300 Subject: [PATCH 041/157] Move store manipulation to its own file --- src/common/buffer.rs | 24 -------- src/common/mod.rs | 4 +- src/common/store.rs | 97 +++++++++++++++++++++++++++++++ src/core.rs | 36 +++++------- src/data/mod.rs | 9 +-- src/lib.rs | 4 +- src/oplog/mod.rs | 44 ++++++-------- src/prelude.rs | 4 +- src/storage_v10/mod.rs | 126 +++++++++++++++++++++++----------------- src/tree/merkle_tree.rs | 20 +++---- 10 files changed, 221 insertions(+), 147 deletions(-) delete mode 100644 src/common/buffer.rs create mode 100644 src/common/store.rs diff --git a/src/common/buffer.rs b/src/common/buffer.rs deleted file mode 100644 index 07e3c64a..00000000 --- a/src/common/buffer.rs +++ /dev/null @@ -1,24 +0,0 @@ -/// Represents a slice to a known buffer. Useful for indicating changes that should be made to random -/// access storages. Data value of None, indicates that the should be truncated to the start -/// position. -#[derive(Debug)] -pub struct BufferSlice { - pub(crate) index: u64, - pub(crate) data: Option>, -} - -impl BufferSlice { - pub fn get_data_mut(&self) -> Box<[u8]> { - let data = self.data.as_ref().unwrap(); - let mut buffer = vec![0; data.len()]; - buffer.copy_from_slice(&data); - buffer.into_boxed_slice() - } -} - -/// Represents an instruction to read a known buffer. -#[derive(Debug)] -pub struct BufferSliceInstruction { - pub(crate) index: u64, - pub(crate) len: u64, -} diff --git a/src/common/mod.rs b/src/common/mod.rs index 923f1560..ef84f7df 100644 --- a/src/common/mod.rs +++ b/src/common/mod.rs @@ -1,5 +1,5 @@ -mod buffer; mod node; +mod store; -pub use self::buffer::{BufferSlice, BufferSliceInstruction}; pub use self::node::Node; +pub use self::store::{Store, StoreInfo, StoreInfoInstruction, StoreInfoType}; diff --git a/src/common/store.rs b/src/common/store.rs new file mode 100644 index 00000000..24bcf5da --- /dev/null +++ b/src/common/store.rs @@ -0,0 +1,97 @@ +/// The types of stores that can be created. +#[derive(Debug, Clone, PartialEq)] +pub enum Store { + /// Tree + Tree, + /// Data (block store) + Data, + /// Bitfield + Bitfield, + /// Oplog + Oplog, +} + +/// Information type about a store. +#[derive(Debug)] +pub enum StoreInfoType { + /// Read/write content of the store + Content, + /// Size in bytes of the store. When flushed, truncates to the given index. `data` is `None`. + Size, +} + +/// Piece of information about a store. Useful for indicating changes that should be made to random +/// access storages or information read from them. +#[derive(Debug)] +pub struct StoreInfo { + pub(crate) store: Store, + pub(crate) info_type: StoreInfoType, + pub(crate) index: u64, + pub(crate) length: Option, + pub(crate) data: Option>, + pub(crate) drop: bool, +} + +impl StoreInfo { + pub fn new_content(store: Store, index: u64, data: &[u8]) -> Self { + Self { + store, + info_type: StoreInfoType::Content, + index, + length: Some(data.len() as u64), + data: Some(data.into()), + drop: false, + } + } + + pub fn new_truncate(store: Store, index: u64) -> Self { + Self { + store, + info_type: StoreInfoType::Size, + index, + length: None, + data: None, + drop: true, + } + } + + pub fn new_size(store: Store, index: u64, length: u64) -> Self { + Self { + store, + info_type: StoreInfoType::Size, + index, + length: Some(length), + data: None, + drop: false, + } + } +} + +/// Represents an instruction to obtain information about a store. +#[derive(Debug)] +pub struct StoreInfoInstruction { + pub(crate) store: Store, + pub(crate) info_type: StoreInfoType, + pub(crate) index: u64, + pub(crate) length: Option, +} + +impl StoreInfoInstruction { + pub fn new_content(store: Store, index: u64, length: u64) -> Self { + Self { + store, + info_type: StoreInfoType::Content, + index, + length: Some(length), + } + } + + pub fn new_size(store: Store, index: u64) -> Self { + Self { + store, + info_type: StoreInfoType::Size, + index, + length: None, + } + } +} diff --git a/src/core.rs b/src/core.rs index bd9a5448..b0c8bb71 100644 --- a/src/core.rs +++ b/src/core.rs @@ -1,11 +1,11 @@ //! Hypercore's main abstraction. Exposes an append-only, secure log structure. -pub use crate::storage_v10::{PartialKeypair, Storage, Store}; - use crate::{ + common::Store, crypto::generate_keypair, data::BlockStore, oplog::{Header, Oplog, MAX_OPLOG_ENTRIES_BYTE_SIZE}, + storage_v10::{PartialKeypair, Storage}, tree::MerkleTree, }; use anyhow::Result; @@ -62,14 +62,14 @@ where let oplog_bytes = storage.read_all(Store::Oplog).await?; let oplog_open_outcome = Oplog::open(key_pair.clone(), oplog_bytes)?; storage - .flush_slices(Store::Oplog, &oplog_open_outcome.slices_to_flush) + .flush_infos(&oplog_open_outcome.infos_to_flush) .await?; // Open/create tree - let slice_instructions = - MerkleTree::get_slice_instructions_to_read(&oplog_open_outcome.header.tree); - let slices = storage.read_slices(Store::Tree, slice_instructions).await?; - let tree = MerkleTree::open(&oplog_open_outcome.header.tree, slices)?; + let info_instructions = + MerkleTree::get_info_instructions_to_read(&oplog_open_outcome.header.tree); + let infos = storage.read_infos(info_instructions).await?; + let tree = MerkleTree::open(&oplog_open_outcome.header.tree, infos)?; // Create block store instance let block_store = BlockStore::default(); @@ -101,16 +101,14 @@ where changeset.hash_and_sign(&self.key_pair.public, &secret_key); // Write the received data to the block store - let slice = self + let info = self .block_store .append_batch(batch, batch_length, self.tree.byte_length); - self.storage.flush_slice(Store::Data, slice).await?; + self.storage.flush_info(info).await?; // Append the changeset to the Oplog let outcome = self.oplog.append_changeset(&changeset, false, &self.header); - self.storage - .flush_slices(Store::Oplog, &outcome.slices_to_flush) - .await?; + self.storage.flush_infos(&outcome.infos_to_flush).await?; self.header = outcome.header; // TODO: write bitfield and contiguous length @@ -139,14 +137,12 @@ where async fn flush_bitfield_and_tree_and_oplog(&mut self) -> Result<()> { // TODO: - // let slices = self.bitfield.flush(); - // self.storage.flush_slices(Store::Bitfield, &slices).await?; - // let slices = self.tree.flush(); - // self.storage.flush_slices(Store::Tree, &slices).await?; - let slices_to_flush = self.oplog.flush(&self.header); - self.storage - .flush_slices(Store::Oplog, &slices_to_flush) - .await?; + // let infos = self.bitfield.flush(); + // self.storage.flush_infos(Store::Bitfield, &infos).await?; + // let infos = self.tree.flush(); + // self.storage.flush_infos(Store::Tree, &infos).await?; + let infos_to_flush = self.oplog.flush(&self.header); + self.storage.flush_infos(&infos_to_flush).await?; Ok(()) } } diff --git a/src/data/mod.rs b/src/data/mod.rs index 2d353f31..07aa6069 100644 --- a/src/data/mod.rs +++ b/src/data/mod.rs @@ -1,4 +1,4 @@ -use crate::common::BufferSlice; +use crate::common::{Store, StoreInfo}; /// Block store #[derive(Debug, Default)] @@ -10,14 +10,11 @@ impl BlockStore { batch: &[&[u8]], batch_length: usize, byte_length: u64, - ) -> BufferSlice { + ) -> StoreInfo { let mut buffer: Vec = Vec::with_capacity(batch_length); for data in batch.iter() { buffer.extend_from_slice(data); } - BufferSlice { - index: byte_length, - data: Some(buffer.into_boxed_slice()), - } + StoreInfo::new_content(Store::Data, byte_length, &buffer) } } diff --git a/src/lib.rs b/src/lib.rs index 5d1f67d2..f95f8496 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -70,7 +70,7 @@ mod storage_v10; #[cfg(feature = "v10")] mod tree; -pub use crate::common::Node; +pub use crate::common::{Node, Store}; #[cfg(feature = "v10")] pub use crate::core::Hypercore; pub use crate::crypto::{generate_keypair, sign, verify, Signature}; @@ -84,7 +84,7 @@ pub use crate::replicate::Peer; #[cfg(not(feature = "v10"))] pub use crate::storage::{NodeTrait, PartialKeypair, Storage, Store}; #[cfg(feature = "v10")] -pub use crate::storage_v10::{PartialKeypair, Storage, Store}; +pub use crate::storage_v10::{PartialKeypair, Storage}; pub use ed25519_dalek::{PublicKey, SecretKey}; use std::path::Path; diff --git a/src/oplog/mod.rs b/src/oplog/mod.rs index f21f55c6..38bec57d 100644 --- a/src/oplog/mod.rs +++ b/src/oplog/mod.rs @@ -1,4 +1,4 @@ -use crate::common::BufferSlice; +use crate::common::{Store, StoreInfo}; use crate::compact_encoding::{CompactEncoding, State}; use crate::tree::MerkleTreeChangeset; use crate::{Node, PartialKeypair}; @@ -29,7 +29,7 @@ pub struct Oplog { #[derive(Debug)] pub struct OplogCreateHeaderOutcome { pub header: Header, - pub slices_to_flush: Box<[BufferSlice]>, + pub infos_to_flush: Box<[StoreInfo]>, } /// Oplog open outcome @@ -37,7 +37,7 @@ pub struct OplogCreateHeaderOutcome { pub struct OplogOpenOutcome { pub oplog: Oplog, pub header: Header, - pub slices_to_flush: Box<[BufferSlice]>, + pub infos_to_flush: Box<[StoreInfo]>, } impl OplogOpenOutcome { @@ -45,7 +45,7 @@ impl OplogOpenOutcome { Self { oplog, header: create_header_outcome.header, - slices_to_flush: create_header_outcome.slices_to_flush, + infos_to_flush: create_header_outcome.infos_to_flush, } } } @@ -100,7 +100,7 @@ impl Oplog { Ok(OplogOpenOutcome { oplog, header, - slices_to_flush: Box::new([]), + infos_to_flush: Box::new([]), }) } else if let Some(mut h2_outcome) = h2_outcome { // This shouldn't happen because the first header is saved to the first slot @@ -114,7 +114,7 @@ impl Oplog { Ok(OplogOpenOutcome { oplog, header: h2_outcome.state.decode(&existing), - slices_to_flush: Box::new([]), + infos_to_flush: Box::new([]), }) } else { // There is nothing in the oplog, start from new. @@ -158,21 +158,21 @@ impl Oplog { OplogCreateHeaderOutcome { header, - slices_to_flush: self.append_entries(&[entry], atomic), + infos_to_flush: self.append_entries(&[entry], atomic), } } /// Flushes pending changes, returns buffer slices to write to storage. - pub fn flush(&mut self, header: &Header) -> Box<[BufferSlice]> { - let (new_header_bits, slices_to_flush) = Self::insert_header(header, 0, self.header_bits); + pub fn flush(&mut self, header: &Header) -> Box<[StoreInfo]> { + let (new_header_bits, infos_to_flush) = Self::insert_header(header, 0, self.header_bits); self.entries_byte_length = 0; self.entries_length = 0; self.header_bits = new_header_bits; - slices_to_flush + infos_to_flush } /// Appends a batch of entries to the Oplog. - fn append_entries(&mut self, batch: &[Entry], atomic: bool) -> Box<[BufferSlice]> { + fn append_entries(&mut self, batch: &[Entry], atomic: bool) -> Box<[StoreInfo]> { let len = batch.len(); let header_bit = self.get_current_header_bit(); // Leave room for leaders @@ -202,18 +202,14 @@ impl Oplog { self.entries_length += len as u64; self.entries_byte_length += buffer.len() as u64; - vec![BufferSlice { - index, - data: Some(buffer), - }] - .into_boxed_slice() + vec![StoreInfo::new_content(Store::Oplog, index, &buffer)].into_boxed_slice() } fn new(key_pair: PartialKeypair) -> OplogOpenOutcome { let entries_length: u64 = 0; let entries_byte_length: u64 = 0; let header = Header::new(key_pair); - let (header_bits, slices_to_flush) = + let (header_bits, infos_to_flush) = Self::insert_header(&header, entries_byte_length, INITIAL_HEADER_BITS); let oplog = Oplog { header_bits, @@ -224,7 +220,7 @@ impl Oplog { oplog, OplogCreateHeaderOutcome { header, - slices_to_flush, + infos_to_flush, }, ) } @@ -233,7 +229,7 @@ impl Oplog { header: &Header, entries_byte_length: u64, current_header_bits: [bool; 2], - ) -> ([bool; 2], Box<[BufferSlice]>) { + ) -> ([bool; 2], Box<[StoreInfo]>) { // The first 8 bytes will be filled with `prepend_leader`. let data_start_index: usize = 8; let mut state = State::new_with_start_and_end(data_start_index, data_start_index); @@ -274,14 +270,8 @@ impl Oplog { ( new_header_bits, vec![ - BufferSlice { - index: oplog_slot as u64, - data: Some(buffer), - }, - BufferSlice { - index: truncate_index, - data: None, - }, + StoreInfo::new_content(Store::Oplog, oplog_slot as u64, &buffer), + StoreInfo::new_truncate(Store::Oplog, truncate_index), ] .into_boxed_slice(), ) diff --git a/src/prelude.rs b/src/prelude.rs index 618ad483..3d024215 100644 --- a/src/prelude.rs +++ b/src/prelude.rs @@ -9,10 +9,10 @@ pub use crate::feed::Feed; // pub use feed_builder::FeedBuilder; #[cfg(not(feature = "v10"))] -pub use crate::common::Node; +pub use crate::common::{Node, Store}; #[cfg(feature = "v10")] pub use crate::core::Hypercore; #[cfg(not(feature = "v10"))] pub use crate::storage::{NodeTrait, Storage, Store}; #[cfg(feature = "v10")] -pub use crate::storage_v10::{PartialKeypair, Storage, Store}; +pub use crate::storage_v10::{PartialKeypair, Storage}; diff --git a/src/storage_v10/mod.rs b/src/storage_v10/mod.rs index 4780a9f5..4068204e 100644 --- a/src/storage_v10/mod.rs +++ b/src/storage_v10/mod.rs @@ -10,7 +10,7 @@ use random_access_storage::RandomAccess; use std::fmt::Debug; use std::path::PathBuf; -use crate::common::{BufferSlice, BufferSliceInstruction}; +use crate::common::{Store, StoreInfo, StoreInfoInstruction, StoreInfoType}; /// Key pair where for read-only hypercores the secret key can also be missing. #[derive(Debug)] @@ -37,19 +37,6 @@ impl Clone for PartialKeypair { } } -/// The types of stores that can be created. -#[derive(Debug)] -pub enum Store { - /// Tree - Tree, - /// Data - Data, - /// Bitfield - Bitfield, - /// Oplog - Oplog, -} - /// Save data to a desired storage backend. #[derive(Debug)] pub struct Storage @@ -91,63 +78,96 @@ where /// Read fully a store. pub async fn read_all(&mut self, store: Store) -> Result> { - let storage = self.get_random_access(store); + let storage = self.get_random_access(&store); let len = storage.len().await.map_err(|e| anyhow!(e))?; let buf = storage.read(0, len).await.map_err(|e| anyhow!(e))?; Ok(buf.into_boxed_slice()) } - /// Read slices from a store based on given instructions - pub async fn read_slices( + /// Read infos from a store based on given instructions + pub async fn read_infos( &mut self, - store: Store, - slice_instructions: Box<[BufferSliceInstruction]>, - ) -> Result> { - let storage = self.get_random_access(store); - - let mut slices: Vec = Vec::with_capacity(slice_instructions.len()); - for instruction in slice_instructions.iter() { - let buf = storage - .read(instruction.index, instruction.len) - .await - .map_err(|e| anyhow!(e))?; - slices.push(BufferSlice { - index: instruction.index, - data: Some(buf.into_boxed_slice()), - }); + info_instructions: Box<[StoreInfoInstruction]>, + ) -> Result> { + if info_instructions.is_empty() { + return Ok(vec![].into_boxed_slice()); } - Ok(slices.into_boxed_slice()) + let mut current_store: Store = info_instructions[0].store.clone(); + let mut storage = self.get_random_access(¤t_store); + let mut infos: Vec = Vec::with_capacity(info_instructions.len()); + for instruction in info_instructions.iter() { + if instruction.store != current_store { + current_store = instruction.store.clone(); + storage = self.get_random_access(¤t_store); + } + match instruction.info_type { + StoreInfoType::Content => { + let buf = storage + .read(instruction.index, instruction.length.unwrap()) + .await + .map_err(|e| anyhow!(e))?; + infos.push(StoreInfo::new_content( + instruction.store.clone(), + instruction.index, + &buf, + )); + } + StoreInfoType::Size => { + let length = storage.len().await.map_err(|e| anyhow!(e))?; + infos.push(StoreInfo::new_size( + instruction.store.clone(), + instruction.index, + length - instruction.index, + )); + } + } + } + Ok(infos.into_boxed_slice()) } - /// Flush slice to storage. Convenience method to `flush_slices`. - pub async fn flush_slice(&mut self, store: Store, slice: BufferSlice) -> Result<()> { - self.flush_slices(store, &[slice]).await + /// Flush info to storage. Convenience method to `flush_infos`. + pub async fn flush_info(&mut self, slice: StoreInfo) -> Result<()> { + self.flush_infos(&[slice]).await } - /// Flush slices to storage. Either writes directly to a random access storage or truncates the storage - /// to the length of given index. - pub async fn flush_slices(&mut self, store: Store, slices: &[BufferSlice]) -> Result<()> { - let storage = self.get_random_access(store); - for slice in slices.iter() { - match &slice.data { - Some(data) => { - storage - .write(slice.index, &data.to_vec()) - .await - .map_err(|e| anyhow!(e))?; + /// Flush infos to storage + pub async fn flush_infos(&mut self, infos: &[StoreInfo]) -> Result<()> { + if infos.is_empty() { + return Ok(()); + } + let mut current_store: Store = infos[0].store.clone(); + let mut storage = self.get_random_access(¤t_store); + for info in infos.iter() { + if info.store != current_store { + current_store = info.store.clone(); + storage = self.get_random_access(¤t_store); + } + match info.info_type { + StoreInfoType::Content => { + if !info.drop { + if let Some(data) = &info.data { + storage + .write(info.index, &data.to_vec()) + .await + .map_err(|e| anyhow!(e))?; + } + } else { + unimplemented!("Deleting not implemented yet") + } } - None => { - storage - .truncate(slice.index) - .await - .map_err(|e| anyhow!(e))?; + StoreInfoType::Size => { + if info.drop { + storage.truncate(info.index).await.map_err(|e| anyhow!(e))?; + } else { + panic!("Flushing a size that isn't drop, is not supported"); + } } } } Ok(()) } - fn get_random_access(&mut self, store: Store) -> &mut T { + fn get_random_access(&mut self, store: &Store) -> &mut T { match store { Store::Tree => &mut self.tree, Store::Data => &mut self.data, diff --git a/src/tree/merkle_tree.rs b/src/tree/merkle_tree.rs index 627c756a..dc31ef35 100644 --- a/src/tree/merkle_tree.rs +++ b/src/tree/merkle_tree.rs @@ -3,8 +3,9 @@ use anyhow::Result; use crate::compact_encoding::State; use crate::oplog::HeaderTree; +use crate::Store; use crate::{ - common::{BufferSlice, BufferSliceInstruction}, + common::{StoreInfo, StoreInfoInstruction}, Node, }; @@ -26,25 +27,22 @@ impl MerkleTree { /// Gets instructions to slices that should be read from storage /// based on `HeaderTree` `length` field. Call this before /// calling `open`. - pub fn get_slice_instructions_to_read( - header_tree: &HeaderTree, - ) -> Box<[BufferSliceInstruction]> { + pub fn get_info_instructions_to_read(header_tree: &HeaderTree) -> Box<[StoreInfoInstruction]> { let root_indices = get_root_indices(&header_tree.length); root_indices .iter() - .map(|&index| BufferSliceInstruction { - index: NODE_SIZE * index, - len: NODE_SIZE, + .map(|&index| { + StoreInfoInstruction::new_content(Store::Tree, NODE_SIZE * index, NODE_SIZE) }) - .collect::>() + .collect::>() .into_boxed_slice() } - /// Opens MerkleTree, based on read byte slices. Call `get_slice_instructions_to_read` + /// Opens MerkleTree, based on read byte slices. Call `get_info_instructions_to_read` /// before calling this to find out which slices to read. The given slices - /// need to be in the same order as the instructions from `get_slice_instructions_to_read`! - pub fn open(header_tree: &HeaderTree, slices: Box<[BufferSlice]>) -> Result { + /// need to be in the same order as the instructions from `get_info_instructions_to_read`! + pub fn open(header_tree: &HeaderTree, slices: Box<[StoreInfo]>) -> Result { let root_indices = get_root_indices(&header_tree.length); let mut roots: Vec = Vec::with_capacity(slices.len()); From e448217319de95f18ec7c9813f3df94d807f597c Mon Sep 17 00:00:00 2001 From: Timo Tiuraniemi Date: Wed, 7 Sep 2022 10:27:58 +0300 Subject: [PATCH 042/157] Use intmap in place of big-sparse-array --- Cargo.toml | 3 ++- src/bitfield_v10/dynamic.rs | 16 +++++++++++++++- 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 0479a5f8..990bfb3d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -42,6 +42,7 @@ bitfield-rle = "0.2.0" futures = "0.3.4" async-std = "1.5.0" crc32fast = "1.3.2" +intmap = { version = "2.0.0", optional = true} [target.'cfg(not(target_arch = "wasm32"))'.dependencies] random-access-disk = "2.0.0" @@ -57,7 +58,7 @@ sha2 = "0.10.2" [features] default = ["v10"] # v10 version of the crate, without this, v9 is built -v10 = [] +v10 = ["intmap"] # Used only in interoperability tests under tests/js-interop which use the javascript version of hypercore # to verify that this crate works. To run them, use: # cargo test --features js-interop-tests diff --git a/src/bitfield_v10/dynamic.rs b/src/bitfield_v10/dynamic.rs index f48e2b09..b3ff142d 100644 --- a/src/bitfield_v10/dynamic.rs +++ b/src/bitfield_v10/dynamic.rs @@ -1,3 +1,17 @@ +use super::fixed::FixedBitfield; + +const DYNAMIC_BITFIELD_PAGE_SIZE: usize = 32768; + /// Dynamic sized bitfield, uses a collection of `FixeddBitfield` elements. #[derive(Debug)] -pub struct DynamicBitfield {} +pub struct DynamicBitfield { + pages: intmap::IntMap, +} + +impl DynamicBitfield { + pub fn new() -> Self { + Self { + pages: intmap::IntMap::new(), + } + } +} From 2b8126e38806295b2c3e087cd7f6ce0df8c9adc1 Mon Sep 17 00:00:00 2001 From: Timo Tiuraniemi Date: Wed, 7 Sep 2022 10:58:53 +0300 Subject: [PATCH 043/157] Removed read_all method, use StoreInfoInstruction for that too --- src/common/store.rs | 9 +++++++++ src/core.rs | 11 ++++++++--- src/storage_v10/mod.rs | 34 +++++++++++++++++++++++----------- 3 files changed, 40 insertions(+), 14 deletions(-) diff --git a/src/common/store.rs b/src/common/store.rs index 24bcf5da..50dd7d71 100644 --- a/src/common/store.rs +++ b/src/common/store.rs @@ -86,6 +86,15 @@ impl StoreInfoInstruction { } } + pub fn new_all_content(store: Store) -> Self { + Self { + store, + info_type: StoreInfoType::Content, + index: 0, + length: None, + } + } + pub fn new_size(store: Store, index: u64) -> Self { Self { store, diff --git a/src/core.rs b/src/core.rs index b0c8bb71..18efd7f4 100644 --- a/src/core.rs +++ b/src/core.rs @@ -1,7 +1,7 @@ //! Hypercore's main abstraction. Exposes an append-only, secure log structure. use crate::{ - common::Store, + common::{Store, StoreInfoInstruction}, crypto::generate_keypair, data::BlockStore, oplog::{Header, Oplog, MAX_OPLOG_ENTRIES_BYTE_SIZE}, @@ -59,7 +59,12 @@ where key_pair: PartialKeypair, ) -> Result> { // Open/create oplog - let oplog_bytes = storage.read_all(Store::Oplog).await?; + let oplog_bytes = storage + .read_info(StoreInfoInstruction::new_all_content(Store::Oplog)) + .await? + .data + .expect("Did not receive data"); + let oplog_open_outcome = Oplog::open(key_pair.clone(), oplog_bytes)?; storage .flush_infos(&oplog_open_outcome.infos_to_flush) @@ -68,7 +73,7 @@ where // Open/create tree let info_instructions = MerkleTree::get_info_instructions_to_read(&oplog_open_outcome.header.tree); - let infos = storage.read_infos(info_instructions).await?; + let infos = storage.read_infos(&info_instructions).await?; let tree = MerkleTree::open(&oplog_open_outcome.header.tree, infos)?; // Create block store instance diff --git a/src/storage_v10/mod.rs b/src/storage_v10/mod.rs index 4068204e..4013c696 100644 --- a/src/storage_v10/mod.rs +++ b/src/storage_v10/mod.rs @@ -76,21 +76,29 @@ where Ok(instance) } - /// Read fully a store. - pub async fn read_all(&mut self, store: Store) -> Result> { - let storage = self.get_random_access(&store); - let len = storage.len().await.map_err(|e| anyhow!(e))?; - let buf = storage.read(0, len).await.map_err(|e| anyhow!(e))?; - Ok(buf.into_boxed_slice()) + /// Read info from store based on given instruction. Convenience method to `read_infos`. + pub async fn read_info(&mut self, info_instruction: StoreInfoInstruction) -> Result { + let mut infos = self.read_infos_to_vec(&[info_instruction]).await?; + Ok(infos + .pop() + .expect("Should have gotten one info with one instruction")) } - /// Read infos from a store based on given instructions + /// Read infos from stores based on given instructions pub async fn read_infos( &mut self, - info_instructions: Box<[StoreInfoInstruction]>, + info_instructions: &[StoreInfoInstruction], ) -> Result> { + let infos = self.read_infos_to_vec(info_instructions).await?; + Ok(infos.into_boxed_slice()) + } + + async fn read_infos_to_vec( + &mut self, + info_instructions: &[StoreInfoInstruction], + ) -> Result> { if info_instructions.is_empty() { - return Ok(vec![].into_boxed_slice()); + return Ok(vec![]); } let mut current_store: Store = info_instructions[0].store.clone(); let mut storage = self.get_random_access(¤t_store); @@ -102,8 +110,12 @@ where } match instruction.info_type { StoreInfoType::Content => { + let length = match instruction.length { + Some(length) => length, + None => storage.len().await.map_err(|e| anyhow!(e))?, + }; let buf = storage - .read(instruction.index, instruction.length.unwrap()) + .read(instruction.index, length) .await .map_err(|e| anyhow!(e))?; infos.push(StoreInfo::new_content( @@ -122,7 +134,7 @@ where } } } - Ok(infos.into_boxed_slice()) + Ok(infos) } /// Flush info to storage. Convenience method to `flush_infos`. From f8c3e39ea726d49bf07bf3b03371fa56bd1c88a6 Mon Sep 17 00:00:00 2001 From: Timo Tiuraniemi Date: Wed, 7 Sep 2022 11:31:01 +0300 Subject: [PATCH 044/157] Open bitfield --- src/bitfield_v10/dynamic.rs | 20 ++++++++++++++++++-- src/core.rs | 22 ++++++++++++++++++---- 2 files changed, 36 insertions(+), 6 deletions(-) diff --git a/src/bitfield_v10/dynamic.rs b/src/bitfield_v10/dynamic.rs index b3ff142d..bc1f09cf 100644 --- a/src/bitfield_v10/dynamic.rs +++ b/src/bitfield_v10/dynamic.rs @@ -1,15 +1,31 @@ +use crate::{ + common::{StoreInfo, StoreInfoInstruction}, + Store, +}; + use super::fixed::FixedBitfield; const DYNAMIC_BITFIELD_PAGE_SIZE: usize = 32768; -/// Dynamic sized bitfield, uses a collection of `FixeddBitfield` elements. +/// Dynamic sized bitfield, uses a map of `FixedBitfield` elements. +/// See: +/// https://github.com/hypercore-protocol/hypercore/blob/master/lib/bitfield.js +/// for reference. #[derive(Debug)] pub struct DynamicBitfield { pages: intmap::IntMap, } impl DynamicBitfield { - pub fn new() -> Self { + /// Gets info instruction to read based on the bitfield store length + pub fn get_info_instruction_to_read(bitfield_store_length: u64) -> StoreInfoInstruction { + // Read only multiples of 4 bytes. Javascript: + // const size = st.size - (st.size & 3) + let length = bitfield_store_length - (bitfield_store_length & 3); + StoreInfoInstruction::new_content(Store::Bitfield, 0, length) + } + + pub fn open(info: StoreInfo) -> Self { Self { pages: intmap::IntMap::new(), } diff --git a/src/core.rs b/src/core.rs index 18efd7f4..67ee82fd 100644 --- a/src/core.rs +++ b/src/core.rs @@ -1,6 +1,7 @@ //! Hypercore's main abstraction. Exposes an append-only, secure log structure. use crate::{ + bitfield_v10::Bitfield, common::{Store, StoreInfoInstruction}, crypto::generate_keypair, data::BlockStore, @@ -23,8 +24,7 @@ where pub(crate) oplog: Oplog, pub(crate) tree: MerkleTree, pub(crate) block_store: BlockStore, - // /// Bitfield to keep track of which data we own. - // pub(crate) bitfield: Bitfield, + pub(crate) bitfield: Bitfield, skip_flush_count: u8, // autoFlush in Javascript header: Header, } @@ -79,12 +79,23 @@ where // Create block store instance let block_store = BlockStore::default(); + // Open bitfield + let bitfield_store_length = storage + .read_info(StoreInfoInstruction::new_size(Store::Bitfield, 0)) + .await? + .length + .expect("Did not get store length with size instruction"); + let info_instruction = Bitfield::get_info_instruction_to_read(bitfield_store_length); + let info = storage.read_info(info_instruction).await?; + let bitfield = Bitfield::open(info); + Ok(Hypercore { key_pair, storage, oplog: oplog_open_outcome.oplog, tree, block_store, + bitfield, skip_flush_count: 0, header: oplog_open_outcome.header, }) @@ -116,8 +127,11 @@ where self.storage.flush_infos(&outcome.infos_to_flush).await?; self.header = outcome.header; - // TODO: write bitfield and contiguous length - // + // Write to bitfield + // TODO + // this.bitfield.setRange(batch.ancestors, batch.length - batch.ancestors, true) + + // TODO: contiguous length if self.should_flush_bitfield_and_tree_and_oplog() { self.flush_bitfield_and_tree_and_oplog().await?; } From abdfae3749552276ea70c18076a717c50e95f514 Mon Sep 17 00:00:00 2001 From: Timo Tiuraniemi Date: Wed, 7 Sep 2022 12:42:59 +0300 Subject: [PATCH 045/157] Added setting range to fixed bitfield --- src/bitfield_v10/fixed.rs | 95 ++++++++++++++++++++++++++++++++++++--- 1 file changed, 90 insertions(+), 5 deletions(-) diff --git a/src/bitfield_v10/fixed.rs b/src/bitfield_v10/fixed.rs index 70e04373..d387c8ae 100644 --- a/src/bitfield_v10/fixed.rs +++ b/src/bitfield_v10/fixed.rs @@ -1,6 +1,6 @@ const FIXED_BITFIELD_LENGTH: usize = 1024; // u32 has 4 bytes and a byte has 8 bits -const FIXED_BITFIELD_BITS_PER_ELEM: u64 = 4 * 8; +const FIXED_BITFIELD_BITS_PER_ELEM: u32 = 4 * 8; use std::convert::TryInto; /// Fixed size bitfield @@ -21,7 +21,7 @@ impl FixedBitfield { } } - pub fn get(&self, index: u64) -> bool { + pub fn get(&self, index: u32) -> bool { let n = FIXED_BITFIELD_BITS_PER_ELEM; let offset = index & (n - 1); let i: usize = ((index - offset) / n) @@ -30,7 +30,7 @@ impl FixedBitfield { self.bitfield[i] & (1 << offset) != 0 } - pub fn set(&mut self, index: u64, value: bool) -> bool { + pub fn set(&mut self, index: u32, value: bool) -> bool { let n = FIXED_BITFIELD_BITS_PER_ELEM; let offset = index & (n - 1); let i: usize = ((index - offset) / n) @@ -50,29 +50,114 @@ impl FixedBitfield { self.bitfield[i] ^= mask; true } + + pub fn set_range(&mut self, start: u32, end: u32, value: bool) -> bool { + let n = FIXED_BITFIELD_BITS_PER_ELEM; + + let mut remaining: i64 = (end - start).into(); + let mut offset = start & (n - 1); + let mut i: usize = ((start - offset) / n).try_into().unwrap(); + + let mut changed = false; + + while remaining > 0 { + let base: u32 = 2; + let power: u32 = std::cmp::min(remaining, (n - offset).into()) + .try_into() + .unwrap(); + let mask_seed = if power == 32 { + // Go directly to this maximum value as the below + // calculation overflows as 1 is subtracted after + // the power. + u32::MAX + } else { + base.pow(power) - 1 + }; + let mask: u32 = mask_seed << offset; + + if value { + if (self.bitfield[i] & mask) != mask { + self.bitfield[i] |= mask; + changed = true; + } + } else { + if (self.bitfield[i] & mask) != 0 { + self.bitfield[i] &= !mask; + changed = true; + } + } + + remaining -= (n - offset) as i64; + offset = 0; + i += 1; + } + + changed + } } #[cfg(test)] mod tests { use super::*; + fn assert_value_range(bitfield: &FixedBitfield, start: u32, end: u32, value: bool) { + for i in start..end { + assert_eq!(bitfield.get(i), value); + } + } + #[test] fn bitfield_fixed_get_and_set() { let mut bitfield = FixedBitfield::new(0); - assert_eq!(bitfield.get(0), false); + assert_value_range(&bitfield, 0, 9, false); bitfield.set(0, true); assert_eq!(bitfield.get(0), true); - assert_eq!(bitfield.get(31), false); + assert_value_range(&bitfield, 1, 64, false); bitfield.set(31, true); assert_eq!(bitfield.get(31), true); + assert_value_range(&bitfield, 32, 64, false); assert_eq!(bitfield.get(32), false); bitfield.set(32, true); assert_eq!(bitfield.get(32), true); + assert_value_range(&bitfield, 33, 64, false); + assert_value_range(&bitfield, 32760, 32768, false); assert_eq!(bitfield.get(32767), false); bitfield.set(32767, true); assert_eq!(bitfield.get(32767), true); + assert_value_range(&bitfield, 32760, 32767, false); + } + + #[test] + fn bitfield_set_range() { + let mut bitfield = FixedBitfield::new(0); + bitfield.set_range(0, 2, true); + assert_value_range(&bitfield, 0, 2, true); + assert_value_range(&bitfield, 3, 64, false); + + bitfield.set_range(2, 5, true); + assert_value_range(&bitfield, 0, 5, true); + assert_value_range(&bitfield, 6, 64, false); + + bitfield.set_range(1, 4, false); + assert_eq!(bitfield.get(0), true); + assert_value_range(&bitfield, 1, 4, false); + assert_value_range(&bitfield, 4, 5, true); + assert_value_range(&bitfield, 6, 64, false); + + bitfield.set_range(31, 31000, true); + assert_value_range(&bitfield, 6, 31, false); + assert_value_range(&bitfield, 31, 100, true); + assert_value_range(&bitfield, 30050, 31000, true); + assert_value_range(&bitfield, 31000, 31050, false); + + bitfield.set_range(32750, 32768, true); + assert_value_range(&bitfield, 32750, 32768, true); + + bitfield.set_range(32765, 32768, false); + assert_value_range(&bitfield, 32750, 32765, true); + assert_value_range(&bitfield, 32765, 32768, false); } } From f7838ecc6eedb7485ac614ade4ac87d2f76cf70c Mon Sep 17 00:00:00 2001 From: Timo Tiuraniemi Date: Wed, 7 Sep 2022 14:40:18 +0300 Subject: [PATCH 046/157] Implement dynamic bitfield --- src/bitfield_v10/dynamic.rs | 163 +++++++++++++++++++++++++++++++++++- src/bitfield_v10/fixed.rs | 65 +++++++------- 2 files changed, 194 insertions(+), 34 deletions(-) diff --git a/src/bitfield_v10/dynamic.rs b/src/bitfield_v10/dynamic.rs index bc1f09cf..72646a7a 100644 --- a/src/bitfield_v10/dynamic.rs +++ b/src/bitfield_v10/dynamic.rs @@ -1,9 +1,9 @@ +use super::fixed::FixedBitfield; use crate::{ common::{StoreInfo, StoreInfoInstruction}, Store, }; - -use super::fixed::FixedBitfield; +use std::{cell::RefCell, convert::TryInto}; const DYNAMIC_BITFIELD_PAGE_SIZE: usize = 32768; @@ -13,7 +13,8 @@ const DYNAMIC_BITFIELD_PAGE_SIZE: usize = 32768; /// for reference. #[derive(Debug)] pub struct DynamicBitfield { - pages: intmap::IntMap, + pages: intmap::IntMap>, + unflushed: Vec, } impl DynamicBitfield { @@ -28,6 +29,162 @@ impl DynamicBitfield { pub fn open(info: StoreInfo) -> Self { Self { pages: intmap::IntMap::new(), + unflushed: vec![], + } + } + + pub fn get(&self, index: u64) -> bool { + let j = index & (DYNAMIC_BITFIELD_PAGE_SIZE as u64 - 1); + let i = (index - j) / DYNAMIC_BITFIELD_PAGE_SIZE as u64; + + if !self.pages.contains_key(i) { + false + } else { + let p = self.pages.get(i).unwrap().borrow(); + p.get(j.try_into().expect("Index should have fit into u32")) + } + } + + pub fn set(&mut self, index: u64, value: bool) -> bool { + let j = index & (DYNAMIC_BITFIELD_PAGE_SIZE as u64 - 1); + let i = (index - j) / DYNAMIC_BITFIELD_PAGE_SIZE as u64; + + if !self.pages.contains_key(i) { + self.pages.insert(i, RefCell::new(FixedBitfield::new(i))); + } + + let mut p = self.pages.get_mut(i).unwrap().borrow_mut(); + let changed: bool = p.set(j.try_into().expect("Index should have fit into u32"), value); + + if changed && !p.dirty { + p.dirty = true; + self.unflushed.push(i); + } + changed + } + + pub fn set_range(&mut self, start: u64, length: u64, value: bool) { + let mut j = start & (DYNAMIC_BITFIELD_PAGE_SIZE as u64 - 1); + let mut i = (start - j) / (DYNAMIC_BITFIELD_PAGE_SIZE as u64); + let mut length = length; + + while length > 0 { + if !self.pages.contains_key(i) { + self.pages.insert(i, RefCell::new(FixedBitfield::new(i))); + } + let mut p = self.pages.get_mut(i).unwrap().borrow_mut(); + + let end = std::cmp::min(j + length, DYNAMIC_BITFIELD_PAGE_SIZE as u64); + + let range_start: u32 = j + .try_into() + .expect("Range start should have fit into a u32"); + let range_end: u32 = (end - j) + .try_into() + .expect("Range end should have fit into a u32"); + + let changed = p.set_range(range_start, range_end, value); + if changed && !p.dirty { + p.dirty = true; + self.unflushed.push(i); + } + + j = 0; + i += 1; + length -= range_end as u64; } } } + +#[cfg(test)] +mod tests { + use super::*; + + fn assert_value_range(bitfield: &DynamicBitfield, start: u64, length: u64, value: bool) { + for i in start..start + length { + assert_eq!(bitfield.get(i), value); + } + } + + #[test] + fn bitfield_dynamic_get_and_set() { + let mut bitfield = DynamicBitfield::open(StoreInfo::new_content(Store::Bitfield, 0, &[])); + assert_value_range(&bitfield, 0, 9, false); + bitfield.set(0, true); + assert_eq!(bitfield.get(0), true); + + assert_value_range(&bitfield, 1, 63, false); + bitfield.set(31, true); + assert_eq!(bitfield.get(31), true); + + assert_value_range(&bitfield, 32, 32, false); + assert_eq!(bitfield.get(32), false); + bitfield.set(32, true); + assert_eq!(bitfield.get(32), true); + assert_value_range(&bitfield, 33, 31, false); + + assert_value_range(&bitfield, 32760, 8, false); + assert_eq!(bitfield.get(32767), false); + bitfield.set(32767, true); + assert_eq!(bitfield.get(32767), true); + assert_value_range(&bitfield, 32760, 7, false); + + // Now for over one fixed bitfield values + bitfield.set(32768, true); + assert_value_range(&bitfield, 32767, 2, true); + assert_value_range(&bitfield, 32769, 9, false); + + bitfield.set(10000000, true); + assert_eq!(bitfield.get(10000000), true); + assert_value_range(&bitfield, 9999990, 10, false); + assert_value_range(&bitfield, 10000001, 9, false); + } + + #[test] + fn bitfield_dynamic_set_range() { + let mut bitfield = DynamicBitfield::open(StoreInfo::new_content(Store::Bitfield, 0, &[])); + bitfield.set_range(0, 2, true); + assert_value_range(&bitfield, 0, 2, true); + assert_value_range(&bitfield, 3, 61, false); + + bitfield.set_range(2, 3, true); + assert_value_range(&bitfield, 0, 5, true); + assert_value_range(&bitfield, 5, 59, false); + + bitfield.set_range(1, 3, false); + assert_eq!(bitfield.get(0), true); + assert_value_range(&bitfield, 1, 3, false); + assert_value_range(&bitfield, 4, 1, true); + assert_value_range(&bitfield, 5, 59, false); + + bitfield.set_range(30, 30070, true); + assert_value_range(&bitfield, 5, 25, false); + assert_value_range(&bitfield, 30, 100, true); + assert_value_range(&bitfield, 30050, 50, true); + assert_value_range(&bitfield, 31000, 50, false); + + bitfield.set_range(32750, 18, true); + assert_value_range(&bitfield, 32750, 18, true); + + bitfield.set_range(32765, 3, false); + assert_value_range(&bitfield, 32750, 15, true); + assert_value_range(&bitfield, 32765, 3, false); + + // Now for over one fixed bitfield values + bitfield.set_range(32765, 15, true); + assert_value_range(&bitfield, 32765, 15, true); + assert_value_range(&bitfield, 32780, 9, false); + bitfield.set_range(32766, 3, false); + assert_value_range(&bitfield, 32766, 3, false); + + bitfield.set_range(10000000, 50, true); + assert_value_range(&bitfield, 9999990, 9, false); + assert_value_range(&bitfield, 10000050, 9, false); + + bitfield.set_range(10000010, 10, false); + assert_value_range(&bitfield, 10000000, 10, true); + assert_value_range(&bitfield, 10000010, 10, false); + assert_value_range(&bitfield, 10000020, 30, true); + assert_value_range(&bitfield, 10000050, 9, false); + } +} diff --git a/src/bitfield_v10/fixed.rs b/src/bitfield_v10/fixed.rs index d387c8ae..df2fd65a 100644 --- a/src/bitfield_v10/fixed.rs +++ b/src/bitfield_v10/fixed.rs @@ -7,9 +7,10 @@ use std::convert::TryInto; /// see: /// https://github.com/holepunchto/bits-to-bytes/blob/main/index.js /// for implementations. -#[derive(Debug)] +#[derive(Debug, Copy, Clone)] pub struct FixedBitfield { pub(crate) parent_index: u64, + pub(crate) dirty: bool, bitfield: [u32; FIXED_BITFIELD_LENGTH], } @@ -17,6 +18,7 @@ impl FixedBitfield { pub fn new(parent_index: u64) -> Self { Self { parent_index, + dirty: false, bitfield: [0; FIXED_BITFIELD_LENGTH], } } @@ -51,10 +53,11 @@ impl FixedBitfield { true } - pub fn set_range(&mut self, start: u32, end: u32, value: bool) -> bool { + pub fn set_range(&mut self, start: u32, length: u32, value: bool) -> bool { + let end: u32 = start + length; let n = FIXED_BITFIELD_BITS_PER_ELEM; - let mut remaining: i64 = (end - start).into(); + let mut remaining: i64 = end as i64 - start as i64; let mut offset = start & (n - 1); let mut i: usize = ((start - offset) / n).try_into().unwrap(); @@ -100,8 +103,8 @@ impl FixedBitfield { mod tests { use super::*; - fn assert_value_range(bitfield: &FixedBitfield, start: u32, end: u32, value: bool) { - for i in start..end { + fn assert_value_range(bitfield: &FixedBitfield, start: u32, length: u32, value: bool) { + for i in start..start + length { assert_eq!(bitfield.get(i), value); } } @@ -113,51 +116,51 @@ mod tests { bitfield.set(0, true); assert_eq!(bitfield.get(0), true); - assert_value_range(&bitfield, 1, 64, false); + assert_value_range(&bitfield, 1, 63, false); bitfield.set(31, true); assert_eq!(bitfield.get(31), true); - assert_value_range(&bitfield, 32, 64, false); + assert_value_range(&bitfield, 32, 32, false); assert_eq!(bitfield.get(32), false); bitfield.set(32, true); assert_eq!(bitfield.get(32), true); - assert_value_range(&bitfield, 33, 64, false); + assert_value_range(&bitfield, 33, 31, false); - assert_value_range(&bitfield, 32760, 32768, false); + assert_value_range(&bitfield, 32760, 8, false); assert_eq!(bitfield.get(32767), false); bitfield.set(32767, true); assert_eq!(bitfield.get(32767), true); - assert_value_range(&bitfield, 32760, 32767, false); + assert_value_range(&bitfield, 32760, 7, false); } #[test] - fn bitfield_set_range() { + fn bitfield_fixed_set_range() { let mut bitfield = FixedBitfield::new(0); bitfield.set_range(0, 2, true); assert_value_range(&bitfield, 0, 2, true); - assert_value_range(&bitfield, 3, 64, false); + assert_value_range(&bitfield, 3, 61, false); - bitfield.set_range(2, 5, true); + bitfield.set_range(2, 3, true); assert_value_range(&bitfield, 0, 5, true); - assert_value_range(&bitfield, 6, 64, false); + assert_value_range(&bitfield, 5, 59, false); - bitfield.set_range(1, 4, false); + bitfield.set_range(1, 3, false); assert_eq!(bitfield.get(0), true); - assert_value_range(&bitfield, 1, 4, false); - assert_value_range(&bitfield, 4, 5, true); - assert_value_range(&bitfield, 6, 64, false); - - bitfield.set_range(31, 31000, true); - assert_value_range(&bitfield, 6, 31, false); - assert_value_range(&bitfield, 31, 100, true); - assert_value_range(&bitfield, 30050, 31000, true); - assert_value_range(&bitfield, 31000, 31050, false); - - bitfield.set_range(32750, 32768, true); - assert_value_range(&bitfield, 32750, 32768, true); - - bitfield.set_range(32765, 32768, false); - assert_value_range(&bitfield, 32750, 32765, true); - assert_value_range(&bitfield, 32765, 32768, false); + assert_value_range(&bitfield, 1, 3, false); + assert_value_range(&bitfield, 4, 1, true); + assert_value_range(&bitfield, 5, 59, false); + + bitfield.set_range(30, 30070, true); + assert_value_range(&bitfield, 5, 25, false); + assert_value_range(&bitfield, 30, 100, true); + assert_value_range(&bitfield, 30050, 50, true); + assert_value_range(&bitfield, 31000, 50, false); + + bitfield.set_range(32750, 18, true); + assert_value_range(&bitfield, 32750, 18, true); + + bitfield.set_range(32765, 3, false); + assert_value_range(&bitfield, 32750, 15, true); + assert_value_range(&bitfield, 32765, 3, false); } } From 2d48fd2cc6511ea98b411ca18a1c12126dbafa0d Mon Sep 17 00:00:00 2001 From: Timo Tiuraniemi Date: Wed, 7 Sep 2022 16:24:06 +0300 Subject: [PATCH 047/157] Bitfield is flushed and identical to JS, same with Oplog --- src/bitfield_v10/dynamic.rs | 44 +++++++++++++++++++++++++++++++++---- src/bitfield_v10/fixed.rs | 40 +++++++++++++++++++++++++++++++-- src/core.rs | 44 ++++++++++++++++++++++++++++++++----- src/oplog/mod.rs | 2 +- 4 files changed, 117 insertions(+), 13 deletions(-) diff --git a/src/bitfield_v10/dynamic.rs b/src/bitfield_v10/dynamic.rs index 72646a7a..f4179354 100644 --- a/src/bitfield_v10/dynamic.rs +++ b/src/bitfield_v10/dynamic.rs @@ -1,4 +1,4 @@ -use super::fixed::FixedBitfield; +use super::fixed::{FixedBitfield, FIXED_BITFIELD_BYTES_LENGTH, FIXED_BITFIELD_LENGTH}; use crate::{ common::{StoreInfo, StoreInfoInstruction}, Store, @@ -27,10 +27,46 @@ impl DynamicBitfield { } pub fn open(info: StoreInfo) -> Self { - Self { - pages: intmap::IntMap::new(), - unflushed: vec![], + let data = info.data.unwrap(); + let resumed = data.len() >= 4; + if resumed { + let mut pages: intmap::IntMap> = intmap::IntMap::new(); + let mut data_index = 0; + while data_index < data.len() { + let parent_index: u64 = (data_index / FIXED_BITFIELD_LENGTH) as u64; + pages.insert( + parent_index, + RefCell::new(FixedBitfield::from_data(parent_index, data_index, &data)), + ); + data_index += FIXED_BITFIELD_LENGTH; + } + Self { + pages, + unflushed: vec![], + } + } else { + Self { + pages: intmap::IntMap::new(), + unflushed: vec![], + } + } + } + + /// Flushes pending changes, returns info slices to write to storage. + pub fn flush(&mut self) -> Box<[StoreInfo]> { + let mut infos_to_flush: Vec = Vec::with_capacity(self.unflushed.len()); + for unflushed_id in &self.unflushed { + let mut p = self.pages.get_mut(*unflushed_id).unwrap().borrow_mut(); + let data = p.to_bytes(); + infos_to_flush.push(StoreInfo::new_content( + Store::Bitfield, + *unflushed_id * data.len() as u64, + &data, + )); + p.dirty = false; } + self.unflushed = vec![]; + infos_to_flush.into_boxed_slice() } pub fn get(&self, index: u64) -> bool { diff --git a/src/bitfield_v10/fixed.rs b/src/bitfield_v10/fixed.rs index df2fd65a..9b8113d0 100644 --- a/src/bitfield_v10/fixed.rs +++ b/src/bitfield_v10/fixed.rs @@ -1,4 +1,5 @@ -const FIXED_BITFIELD_LENGTH: usize = 1024; +pub(crate) const FIXED_BITFIELD_LENGTH: usize = 1024; +pub(crate) const FIXED_BITFIELD_BYTES_LENGTH: usize = FIXED_BITFIELD_LENGTH * 4; // u32 has 4 bytes and a byte has 8 bits const FIXED_BITFIELD_BITS_PER_ELEM: u32 = 4 * 8; use std::convert::TryInto; @@ -7,7 +8,7 @@ use std::convert::TryInto; /// see: /// https://github.com/holepunchto/bits-to-bytes/blob/main/index.js /// for implementations. -#[derive(Debug, Copy, Clone)] +#[derive(Debug)] pub struct FixedBitfield { pub(crate) parent_index: u64, pub(crate) dirty: bool, @@ -23,6 +24,41 @@ impl FixedBitfield { } } + pub fn from_data(parent_index: u64, data_index: usize, data: &[u8]) -> Self { + let mut bitfield = [0; FIXED_BITFIELD_LENGTH]; + if data.len() >= data_index + 4 { + let mut i = data_index; + let limit = std::cmp::min(data_index + FIXED_BITFIELD_BYTES_LENGTH, data.len()) - 4; + while i <= limit { + let value: u32 = ((data[i] as u32) << 0) + | ((data[i + 1] as u32) << 8) + | ((data[i + 2] as u32) << 16) + | ((data[i + 3] as u32) << 24); + bitfield[i / 4] = value; + i += 4; + } + } + Self { + parent_index, + dirty: false, + bitfield, + } + } + + pub fn to_bytes(&self) -> Box<[u8]> { + let mut data: [u8; FIXED_BITFIELD_BYTES_LENGTH] = [0; FIXED_BITFIELD_BYTES_LENGTH]; + let mut i = 0; + for elem in self.bitfield { + let bytes = &elem.to_le_bytes(); + data[i] = bytes[0]; + data[i + 1] = bytes[1]; + data[i + 2] = bytes[2]; + data[i + 3] = bytes[3]; + i += 4; + } + data.into() + } + pub fn get(&self, index: u32) -> bool { let n = FIXED_BITFIELD_BITS_PER_ELEM; let offset = index & (n - 1); diff --git a/src/core.rs b/src/core.rs index 67ee82fd..8245f6e5 100644 --- a/src/core.rs +++ b/src/core.rs @@ -128,10 +128,16 @@ where self.header = outcome.header; // Write to bitfield - // TODO - // this.bitfield.setRange(batch.ancestors, batch.length - batch.ancestors, true) + self.bitfield.set_range( + changeset.ancestors, + changeset.length - changeset.ancestors, + true, + ); - // TODO: contiguous length + // Contiguous length is known only now + self.update_contiguous_length(false, changeset.ancestors, changeset.batch_length); + + // Now ready to flush if self.should_flush_bitfield_and_tree_and_oplog() { self.flush_bitfield_and_tree_and_oplog().await?; } @@ -154,12 +160,38 @@ where } } + fn update_contiguous_length( + &mut self, + bitfield_drop: bool, + bitfield_start: u64, + bitfield_length: u64, + ) { + let end = bitfield_start + bitfield_length; + let mut c = self.header.contiguous_length; + if bitfield_drop { + if c <= end && c > bitfield_start { + c = bitfield_start; + } + } else { + if c <= end && c >= bitfield_start { + c = end; + while self.bitfield.get(c) { + c += 1; + } + } + } + + if c != self.header.contiguous_length { + self.header.contiguous_length = c; + } + } + async fn flush_bitfield_and_tree_and_oplog(&mut self) -> Result<()> { + let infos = self.bitfield.flush(); + self.storage.flush_infos(&infos).await?; // TODO: - // let infos = self.bitfield.flush(); - // self.storage.flush_infos(Store::Bitfield, &infos).await?; // let infos = self.tree.flush(); - // self.storage.flush_infos(Store::Tree, &infos).await?; + // self.storage.flush_infos(&infos).await?; let infos_to_flush = self.oplog.flush(&self.header); self.storage.flush_infos(&infos_to_flush).await?; Ok(()) diff --git a/src/oplog/mod.rs b/src/oplog/mod.rs index 38bec57d..0cc077d4 100644 --- a/src/oplog/mod.rs +++ b/src/oplog/mod.rs @@ -162,7 +162,7 @@ impl Oplog { } } - /// Flushes pending changes, returns buffer slices to write to storage. + /// Flushes pending changes, returns info slices to write to storage. pub fn flush(&mut self, header: &Header) -> Box<[StoreInfo]> { let (new_header_bits, infos_to_flush) = Self::insert_header(header, 0, self.header_bits); self.entries_byte_length = 0; From f26459ac0dc38fb613dd80a7281d8313f2682640 Mon Sep 17 00:00:00 2001 From: Timo Tiuraniemi Date: Thu, 8 Sep 2022 10:13:36 +0300 Subject: [PATCH 048/157] Milestone 2: Able to append to the hypercore and create an bit-for-bit identical storage (bitfield, data, oplog, tree) --- src/common/node.rs | 11 +++ src/core.rs | 8 +- src/tree/merkle_tree.rs | 133 +++++++++++++++++++++++++++++- src/tree/merkle_tree_changeset.rs | 9 ++ 4 files changed, 157 insertions(+), 4 deletions(-) diff --git a/src/common/node.rs b/src/common/node.rs index a04a2903..1ac308ec 100644 --- a/src/common/node.rs +++ b/src/common/node.rs @@ -41,6 +41,17 @@ impl Node { } } + /// Creates a new blank node + pub fn new_blank(index: u64) -> Self { + Self { + index, + hash: vec![0, 32], + length: 0, + parent: 0, + data: None, + } + } + /// Convert a vector to a new instance. /// /// Requires the index at which the buffer was read to be passed. diff --git a/src/core.rs b/src/core.rs index 8245f6e5..aed556e4 100644 --- a/src/core.rs +++ b/src/core.rs @@ -137,6 +137,9 @@ where // Contiguous length is known only now self.update_contiguous_length(false, changeset.ancestors, changeset.batch_length); + // Commit changeset to in-memory tree + self.tree.commit(changeset)?; + // Now ready to flush if self.should_flush_bitfield_and_tree_and_oplog() { self.flush_bitfield_and_tree_and_oplog().await?; @@ -189,9 +192,8 @@ where async fn flush_bitfield_and_tree_and_oplog(&mut self) -> Result<()> { let infos = self.bitfield.flush(); self.storage.flush_infos(&infos).await?; - // TODO: - // let infos = self.tree.flush(); - // self.storage.flush_infos(&infos).await?; + let infos = self.tree.flush(); + self.storage.flush_infos(&infos).await?; let infos_to_flush = self.oplog.flush(&self.header); self.storage.flush_infos(&infos_to_flush).await?; Ok(()) diff --git a/src/tree/merkle_tree.rs b/src/tree/merkle_tree.rs index dc31ef35..41a94597 100644 --- a/src/tree/merkle_tree.rs +++ b/src/tree/merkle_tree.rs @@ -1,5 +1,6 @@ -use anyhow::ensure; use anyhow::Result; +use anyhow::{anyhow, ensure}; +use ed25519_dalek::Signature; use crate::compact_encoding::State; use crate::oplog::HeaderTree; @@ -19,6 +20,10 @@ pub struct MerkleTree { pub(crate) length: u64, pub(crate) byte_length: u64, pub(crate) fork: u64, + pub(crate) signature: Option, + unflushed: intmap::IntMap, + truncated: bool, + truncate_to: u64, } const NODE_SIZE: u64 = 40; @@ -68,6 +73,10 @@ impl MerkleTree { length, byte_length, fork: header_tree.fork, + unflushed: intmap::IntMap::new(), + truncated: false, + truncate_to: 0, + signature: None, }) } @@ -77,6 +86,128 @@ impl MerkleTree { pub fn changeset(&self) -> MerkleTreeChangeset { MerkleTreeChangeset::new(self.length, self.byte_length, self.fork, self.roots.clone()) } + + /// Commit a created changeset to the tree. + pub fn commit(&mut self, changeset: MerkleTreeChangeset) -> Result<()> { + if !self.commitable(&changeset) { + return Err(anyhow!( + "Tree was modified during changeset, refusing to commit" + )); + } + + if changeset.upgraded { + self.commit_truncation(&changeset); + self.roots = changeset.roots; + self.length = changeset.length; + self.byte_length = changeset.byte_length; + self.fork = changeset.fork; + self.signature = changeset + .hash_and_signature + .map(|hash_and_signature| hash_and_signature.1); + } + + for node in changeset.nodes { + self.unflushed.insert(node.index, node); + } + + Ok(()) + } + + pub fn flush(&mut self) -> Box<[StoreInfo]> { + let mut infos_to_flush: Vec = Vec::new(); + if self.truncated { + infos_to_flush.extend(self.flush_truncation()); + } + infos_to_flush.extend(self.flush_nodes()); + infos_to_flush.into_boxed_slice() + } + + fn commitable(&self, changeset: &MerkleTreeChangeset) -> bool { + let correct_length: bool = if changeset.upgraded { + changeset.original_tree_length == self.length + } else { + changeset.original_tree_length <= self.length + }; + changeset.original_tree_fork == self.fork && correct_length + } + + fn commit_truncation(&mut self, changeset: &MerkleTreeChangeset) { + if changeset.ancestors < changeset.original_tree_length { + if changeset.ancestors > 0 { + let head = 2 * changeset.ancestors; + let mut iter = flat_tree::Iterator::new(head - 2); + loop { + // TODO: we should implement a contains() method in the Iterator + // similar to the Javascript + // https://github.com/mafintosh/flat-tree/blob/master/index.js#L152 + // then this would work: + // if iter.contains(head) && iter.index() < head { + let index = iter.index(); + let factor = iter.factor(); + let contains: bool = if head > index { + head < (index + factor / 2) + } else { + if head < index { + head > (index - factor / 2) + } else { + true + } + }; + if contains && index < head { + self.unflushed.insert(index, Node::new_blank(index)); + } + + if iter.offset() == 0 { + break; + } + iter.parent(); + } + } + + self.truncate_to = if self.truncated { + std::cmp::min(self.truncate_to, changeset.ancestors) + } else { + changeset.ancestors + }; + + self.truncated = true; + let mut unflushed_indices_to_delete: Vec = Vec::new(); + for node in self.unflushed.iter() { + if *node.0 >= 2 * changeset.ancestors { + unflushed_indices_to_delete.push(*node.0); + } + } + for index_to_delete in unflushed_indices_to_delete { + self.unflushed.remove(index_to_delete); + } + } + } + + pub fn flush_truncation(&mut self) -> Vec { + let offset = if self.truncate_to == 0 { + 0 + } else { + (self.truncate_to - 1) * 80 + 40 + }; + self.truncate_to = 0; + self.truncated = false; + vec![StoreInfo::new_truncate(Store::Tree, offset)] + } + + pub fn flush_nodes(&mut self) -> Vec { + let mut infos_to_flush: Vec = Vec::with_capacity(self.unflushed.len()); + for node in self.unflushed.values() { + let (mut state, mut buffer) = State::new_with_size(40); + state.encode_u64(node.length, &mut buffer); + state.encode_fixed_32(&node.hash, &mut buffer); + infos_to_flush.push(StoreInfo::new_content( + Store::Tree, + node.index * 40, + &buffer, + )); + } + infos_to_flush + } } fn get_root_indices(header_tree_length: &u64) -> Vec { diff --git a/src/tree/merkle_tree_changeset.rs b/src/tree/merkle_tree_changeset.rs index 161caa41..1e782808 100644 --- a/src/tree/merkle_tree_changeset.rs +++ b/src/tree/merkle_tree_changeset.rs @@ -21,6 +21,11 @@ pub struct MerkleTreeChangeset { pub(crate) roots: Vec, pub(crate) nodes: Vec, pub(crate) hash_and_signature: Option<(Box<[u8]>, Signature)>, + pub(crate) upgraded: bool, + + // Safeguarding values + pub(crate) original_tree_length: u64, + pub(crate) original_tree_fork: u64, } impl MerkleTreeChangeset { @@ -34,6 +39,9 @@ impl MerkleTreeChangeset { roots, nodes: vec![], hash_and_signature: None, + upgraded: false, + original_tree_length: length, + original_tree_fork: fork, } } @@ -48,6 +56,7 @@ impl MerkleTreeChangeset { } pub fn append_root(&mut self, node: Node, iter: &mut flat_tree::Iterator) { + self.upgraded = true; self.length += iter.factor() / 2; self.byte_length += node.length; self.roots.push(node.clone()); From e77343478b88c82e00aec3a79d5cb69d535ca393 Mon Sep 17 00:00:00 2001 From: Timo Tiuraniemi Date: Thu, 8 Sep 2022 10:23:25 +0300 Subject: [PATCH 049/157] Fix compilation with v9 --- src/lib.rs | 4 +++- src/prelude.rs | 5 +++-- tests/js_interop.rs | 1 + 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index f95f8496..33fb281d 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -70,7 +70,9 @@ mod storage_v10; #[cfg(feature = "v10")] mod tree; -pub use crate::common::{Node, Store}; +pub use crate::common::Node; +#[cfg(feature = "v10")] +pub use crate::common::Store; #[cfg(feature = "v10")] pub use crate::core::Hypercore; pub use crate::crypto::{generate_keypair, sign, verify, Signature}; diff --git a/src/prelude.rs b/src/prelude.rs index 3d024215..188c3a02 100644 --- a/src/prelude.rs +++ b/src/prelude.rs @@ -8,8 +8,9 @@ #[cfg(not(feature = "v10"))] pub use crate::feed::Feed; // pub use feed_builder::FeedBuilder; -#[cfg(not(feature = "v10"))] -pub use crate::common::{Node, Store}; +pub use crate::common::Node; +#[cfg(feature = "v10")] +pub use crate::common::Store; #[cfg(feature = "v10")] pub use crate::core::Hypercore; #[cfg(not(feature = "v10"))] diff --git a/tests/js_interop.rs b/tests/js_interop.rs index 8d00cfed..51ca1140 100644 --- a/tests/js_interop.rs +++ b/tests/js_interop.rs @@ -22,6 +22,7 @@ fn init() { #[async_std::test] #[cfg_attr(not(feature = "js_interop_tests"), ignore)] +#[cfg(feature = "v10")] async fn js_interop_js_first() -> Result<()> { init(); let work_dir = prepare_test_set(TEST_SET_JS_FIRST); From af06e79fbeaf4a56152fd554273c5fe871cd684c Mon Sep 17 00:00:00 2001 From: Timo Tiuraniemi Date: Fri, 9 Sep 2022 08:28:23 +0300 Subject: [PATCH 050/157] Use version of random-access-memory with truncate implemented --- Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index 990bfb3d..14a8b4ce 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -32,7 +32,7 @@ memory-pager = "0.9.0" merkle-tree-stream = "0.12.0" pretty-hash = "0.4.1" rand = "0.7.3" -random-access-memory = "2.0.0" +random-access-memory = { git = "https://github.com/ttiurani/random-access-memory", branch = "v10" } random-access-storage = "4.0.0" sha2 = "0.9.2" sleep-parser = "0.8.0" From 7eb4acd31c57dcd2aaf3493620078d8a8dfc271f Mon Sep 17 00:00:00 2001 From: Timo Tiuraniemi Date: Fri, 9 Sep 2022 09:53:18 +0300 Subject: [PATCH 051/157] Added step 3 failing tests --- src/core.rs | 5 +++-- tests/js/interop.js | 35 +++++++++++++++++++++++++++++++++-- tests/js_interop.rs | 38 +++++++++++++++++++++++++++++++------- 3 files changed, 67 insertions(+), 11 deletions(-) diff --git a/src/core.rs b/src/core.rs index aed556e4..ab29821d 100644 --- a/src/core.rs +++ b/src/core.rs @@ -146,8 +146,9 @@ where } Ok(AppendOutcome { - length: 0, - byte_length: 0, + length: self.tree.length, + // TODO: This comes in JS from the block store write result + byte_length: self.tree.byte_length, }) } diff --git a/tests/js/interop.js b/tests/js/interop.js index 3320c004..cfaf495a 100644 --- a/tests/js/interop.js +++ b/tests/js/interop.js @@ -30,18 +30,49 @@ if (process.argv[2] === '1') { step2AppendHelloWorld(process.argv[3]).then(result => { console.log("step2 ready", result); }); +} else if (process.argv[2] === '3'){ + step3iReadAndAppendUnflushed(process.argv[3]).then(result => { + console.log("step3 ready", result); + }); } else { console.error(`Invalid test step {}`, process.argv[2]); process.exit(2); } async function step1Create(testSet) { - let core = new Hypercore(`work/${testSet}`, testKeyPair.publicKey, {keyPair: testKeyPair}); + const core = new Hypercore(`work/${testSet}`, testKeyPair.publicKey, {keyPair: testKeyPair}); await core.close(); }; async function step2AppendHelloWorld(testSet) { const core = new Hypercore(`work/${testSet}`, testKeyPair.publicKey, {keyPair: testKeyPair}); - await core.append(['Hello', 'World']); + const result = await core.append([Buffer.from('Hello'), Buffer.from('World')]); + if (result.length != 2 || result.byteLength != 10) { + throw new Error(`Invalid append result: ${result.length} or ${result.byteLength}`); + } + await core.close(); +}; + +async function step3iReadAndAppendUnflushed(testSet) { + const core = new Hypercore(`work/${testSet}`, testKeyPair.publicKey, {keyPair: testKeyPair}); + const hello = (await core.get(0)).toString(); + const world = (await core.get(1)).toString(); + if (hello != "Hello" || world != "World") { + throw new Error(`Read invalid data from hypercore: ${hello} or ${world}`); + } + let result = await core.append(Buffer.from('first')); + if (result.length != 3 || result.byteLength != 15) { + throw new Error(`Invalid append result: ${result.length} or ${result.byteLength}`); + } + result = await core.append([Buffer.from('second'), Buffer.from('third')]); + if (result.length != 5 || result.byteLength != 26) { + throw new Error(`Invalid append result: ${result.length} or ${result.byteLength}`); + } + const first = (await core.get(2)).toString(); + const second = (await core.get(3)).toString(); + const third = (await core.get(4)).toString(); + if (first != "first" || second != "second" || third != "third") { + throw new Error(`Read invalid data from hypercore: ${first} or ${second} or ${third}`); + } await core.close(); }; diff --git a/tests/js_interop.rs b/tests/js_interop.rs index 51ca1140..0043e1a8 100644 --- a/tests/js_interop.rs +++ b/tests/js_interop.rs @@ -26,11 +26,13 @@ fn init() { async fn js_interop_js_first() -> Result<()> { init(); let work_dir = prepare_test_set(TEST_SET_JS_FIRST); - assert_eq!(step_0_hash(), create_hypercore_hash(&work_dir)); + assert_eq!(create_hypercore_hash(&work_dir), step_0_hash()); js_run_step(1, TEST_SET_JS_FIRST); - assert_eq!(step_1_hash(), create_hypercore_hash(&work_dir)); + assert_eq!(create_hypercore_hash(&work_dir), step_1_hash()); step_2_append_hello_world(&work_dir).await?; - assert_eq!(step_2_hash(), create_hypercore_hash(&work_dir)); + assert_eq!(create_hypercore_hash(&work_dir), step_2_hash()); + js_run_step(3, TEST_SET_JS_FIRST); + assert_eq!(create_hypercore_hash(&work_dir), step_3_hash()); Ok(()) } @@ -40,11 +42,13 @@ async fn js_interop_js_first() -> Result<()> { async fn js_interop_rs_first() -> Result<()> { init(); let work_dir = prepare_test_set(TEST_SET_RS_FIRST); - assert_eq!(step_0_hash(), create_hypercore_hash(&work_dir)); + assert_eq!(create_hypercore_hash(&work_dir), step_0_hash()); step_1_create(&work_dir).await?; - assert_eq!(step_1_hash(), create_hypercore_hash(&work_dir)); + assert_eq!(create_hypercore_hash(&work_dir), step_1_hash()); js_run_step(2, TEST_SET_RS_FIRST); - assert_eq!(step_2_hash(), create_hypercore_hash(&work_dir)); + assert_eq!(create_hypercore_hash(&work_dir), step_2_hash()); + step_3_read_and_append_unflushed(&work_dir).await?; + assert_eq!(create_hypercore_hash(&work_dir), step_3_hash()); Ok(()) } @@ -63,7 +67,18 @@ async fn step_2_append_hello_world(work_dir: &str) -> Result<()> { let key_pair = get_test_key_pair(); let storage = Storage::new_disk(&path, false).await?; let mut hypercore = Hypercore::new_with_key_pair(storage, key_pair).await?; - let _append_outcome = hypercore.append_batch(&[b"Hello", b"World"]).await?; + let append_outcome = hypercore.append_batch(&[b"Hello", b"World"]).await?; + assert_eq!(append_outcome.length, 2); + assert_eq!(append_outcome.byte_length, 10); + Ok(()) +} + +#[cfg(feature = "v10")] +async fn step_3_read_and_append_unflushed(work_dir: &str) -> Result<()> { + let path = Path::new(work_dir).to_owned(); + let key_pair = get_test_key_pair(); + let storage = Storage::new_disk(&path, false).await?; + let mut _hypercore = Hypercore::new_with_key_pair(storage, key_pair).await?; Ok(()) } @@ -93,3 +108,12 @@ fn step_2_hash() -> common::HypercoreHash { tree: Some("8577B24ADC763F65D562CD11204F938229AD47F27915B0821C46A0470B80813A".into()), } } + +fn step_3_hash() -> common::HypercoreHash { + common::HypercoreHash { + bitfield: Some("DEC1593A7456C8C9407B9B8B9C89682DFFF33C3892BCC9D9F06956FEE0A1B949".into()), + data: Some("402670849413BB97FAC3A322FC1EE3DE20F7F0D9C641B0F70BC4C619B3032C50".into()), + oplog: Some("0536819D13798DADFCA9A8607D10DDD14254902FBCC35E97043039E4308869B5".into()), + tree: Some("38788609A8634DC8D34F9AE723F3169ADB20768ACFDFF266A43B7E217750DD1E".into()), + } +} From b14d077d657cab66872fe3d498e019a889152ee6 Mon Sep 17 00:00:00 2001 From: Timo Tiuraniemi Date: Mon, 12 Sep 2022 14:38:38 +0300 Subject: [PATCH 052/157] A preliminary implementation of reading values --- src/common/mod.rs | 2 +- src/common/node.rs | 29 ++++++- src/core.rs | 46 ++++++++++- src/data/mod.rs | 19 ++++- src/tree/merkle_tree.rs | 176 ++++++++++++++++++++++++++++++++++++++-- tests/js_interop.rs | 18 +++- 6 files changed, 277 insertions(+), 13 deletions(-) diff --git a/src/common/mod.rs b/src/common/mod.rs index ef84f7df..4a6c6459 100644 --- a/src/common/mod.rs +++ b/src/common/mod.rs @@ -1,5 +1,5 @@ mod node; mod store; -pub use self::node::Node; +pub use self::node::{Node, NodeByteRange}; pub use self::store::{Store, StoreInfo, StoreInfoInstruction, StoreInfoType}; diff --git a/src/common/node.rs b/src/common/node.rs index 1ac308ec..3849b582 100644 --- a/src/common/node.rs +++ b/src/common/node.rs @@ -15,6 +15,13 @@ use std::io::Cursor; use crate::crypto::Hash; +/// Node byte range +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct NodeByteRange { + pub(crate) index: u64, + pub(crate) length: u64, +} + /// Nodes that are persisted to disk. // TODO: replace `hash: Vec` with `hash: Hash`. This requires patching / // rewriting the Blake2b crate to support `.from_bytes()` to serialize from @@ -26,6 +33,7 @@ pub struct Node { pub(crate) length: u64, pub(crate) parent: u64, pub(crate) data: Option>, + pub(crate) blank: bool, } impl Node { @@ -38,6 +46,7 @@ impl Node { length: length as u64, parent: flat_tree::parent(index), data: Some(Vec::with_capacity(0)), + blank: false, } } @@ -49,6 +58,7 @@ impl Node { length: 0, parent: 0, data: None, + blank: true, } } @@ -65,8 +75,13 @@ impl Node { // TODO: subslice directly, move cursor forward. let capacity = 32; let mut hash = Vec::with_capacity(capacity); + let mut blank = true; for _ in 0..capacity { - hash.push(reader.read_u8()?); + let byte = reader.read_u8()?; + if blank && byte != 0 { + blank = false; + }; + hash.push(byte); } let length = reader.read_u64::()?; @@ -76,6 +91,7 @@ impl Node { index, parent, data: Some(Vec::with_capacity(0)), + blank, }) } @@ -154,13 +170,22 @@ impl From> for Node { NodeKind::Leaf(data) => Some(data.clone()), NodeKind::Parent => None, }; + let hash: Vec = parts.hash().as_bytes().into(); + let mut blank = true; + for byte in &hash { + if *byte != 0 { + blank = false; + break; + } + } Node { index: partial.index(), parent: partial.parent, length: partial.len() as u64, - hash: parts.hash().as_bytes().into(), + hash, data, + blank, } } } diff --git a/src/core.rs b/src/core.rs index ab29821d..d784f8dd 100644 --- a/src/core.rs +++ b/src/core.rs @@ -9,7 +9,8 @@ use crate::{ storage_v10::{PartialKeypair, Storage}, tree::MerkleTree, }; -use anyhow::Result; +use anyhow::{anyhow, Result}; +use futures::future::Either; use random_access_storage::RandomAccess; use std::fmt::Debug; @@ -101,7 +102,12 @@ where }) } - /// Appends a given batch of data blobs to the hypercore. + /// Appends a data slice to the hypercore. + pub async fn append(&mut self, data: &[u8]) -> Result { + self.append_batch(&[data]).await + } + + /// Appends a given batch of data slices to the hypercore. pub async fn append_batch(&mut self, batch: &[&[u8]]) -> Result { let secret_key = match &self.key_pair.secret { Some(key) => key, @@ -152,6 +158,42 @@ where }) } + /// Read value at given index, if any. + pub async fn get(&mut self, index: u64) -> Result>> { + if !self.bitfield.get(index) { + return Ok(None); + } + + // TODO: Figure out a way to generalize this Either processing stack! + let byte_range = match self.tree.byte_range(index, None)? { + Either::Right(byte_range) => byte_range, + Either::Left(instructions) => { + let infos = self.storage.read_infos(&instructions).await?; + match self.tree.byte_range(index, Some(&infos))? { + Either::Right(byte_range) => byte_range, + Either::Left(_) => { + return Err(anyhow!("Could not read byte range")); + } + } + } + }; + + let data = match self.block_store.read(&byte_range, None) { + Either::Right(data) => data, + Either::Left(instruction) => { + let info = self.storage.read_info(instruction).await?; + match self.block_store.read(&byte_range, Some(info)) { + Either::Right(data) => data, + Either::Left(_) => { + return Err(anyhow!("Could not read block storage range")); + } + } + } + }; + + Ok(Some(data.to_vec())) + } + fn should_flush_bitfield_and_tree_and_oplog(&mut self) -> bool { if self.skip_flush_count == 0 || self.oplog.entries_byte_length >= MAX_OPLOG_ENTRIES_BYTE_SIZE diff --git a/src/data/mod.rs b/src/data/mod.rs index 07aa6069..5bfc799d 100644 --- a/src/data/mod.rs +++ b/src/data/mod.rs @@ -1,4 +1,5 @@ -use crate::common::{Store, StoreInfo}; +use crate::common::{NodeByteRange, Store, StoreInfo, StoreInfoInstruction}; +use futures::future::Either; /// Block store #[derive(Debug, Default)] @@ -17,4 +18,20 @@ impl BlockStore { } StoreInfo::new_content(Store::Data, byte_length, &buffer) } + + pub fn read( + &self, + byte_range: &NodeByteRange, + info: Option, + ) -> Either> { + if let Some(info) = info { + Either::Right(info.data.unwrap()) + } else { + Either::Left(StoreInfoInstruction::new_content( + Store::Data, + byte_range.index, + byte_range.length, + )) + } + } } diff --git a/src/tree/merkle_tree.rs b/src/tree/merkle_tree.rs index 41a94597..3d85263a 100644 --- a/src/tree/merkle_tree.rs +++ b/src/tree/merkle_tree.rs @@ -1,7 +1,9 @@ use anyhow::Result; use anyhow::{anyhow, ensure}; use ed25519_dalek::Signature; +use futures::future::Either; +use crate::common::NodeByteRange; use crate::compact_encoding::State; use crate::oplog::HeaderTree; use crate::Store; @@ -44,22 +46,22 @@ impl MerkleTree { .into_boxed_slice() } - /// Opens MerkleTree, based on read byte slices. Call `get_info_instructions_to_read` - /// before calling this to find out which slices to read. The given slices + /// Opens MerkleTree, based on read infos. Call `get_info_instructions_to_read` + /// before calling this to find out which infos to read. The given infos /// need to be in the same order as the instructions from `get_info_instructions_to_read`! - pub fn open(header_tree: &HeaderTree, slices: Box<[StoreInfo]>) -> Result { + pub fn open(header_tree: &HeaderTree, infos: Box<[StoreInfo]>) -> Result { let root_indices = get_root_indices(&header_tree.length); - let mut roots: Vec = Vec::with_capacity(slices.len()); + let mut roots: Vec = Vec::with_capacity(infos.len()); let mut byte_length: u64 = 0; let mut length: u64 = 0; for i in 0..root_indices.len() { let index = root_indices[i]; ensure!( - index == slices[i].index / NODE_SIZE, + index == index_from_info(&infos[i]), "Given slices vector not in the correct order" ); - let data = slices[i].data.as_ref().unwrap(); + let data = infos[i].data.as_ref().unwrap(); let node = node_from_bytes(&index, data); byte_length += node.length; // This is totalSpan in Javascript @@ -113,6 +115,7 @@ impl MerkleTree { Ok(()) } + /// Flush committed made changes to the tree pub fn flush(&mut self) -> Box<[StoreInfo]> { let mut infos_to_flush: Vec = Vec::new(); if self.truncated { @@ -122,6 +125,82 @@ impl MerkleTree { infos_to_flush.into_boxed_slice() } + /// Get storage byte range of given hypercore index + pub fn byte_range( + &self, + hypercore_index: u64, + infos: Option<&[StoreInfo]>, + ) -> Result, NodeByteRange>> { + // Converts a hypercore index into a merkle tree index + let index = 2 * hypercore_index; + + // Check bounds + let head = 2 * self.length; + let compare_index = if index & 1 == 0 { + index + } else { + flat_tree::right_span(index) + }; + if compare_index >= head { + return Err(anyhow!( + "Hypercore index {} is out of bounds", + hypercore_index + )); + } + + // Get nodes out of incoming infos + let nodes: Vec = match infos { + Some(infos) => infos + .iter() + .map(|info| node_from_bytes(&index_from_info(&info), info.data.as_ref().unwrap())) + .collect(), + None => vec![], + }; + + // Start with getting the requested node, which will get the length + // of the byte range + let length_result = self.get_node(index, &nodes)?; + + // As for the offset, that might require a lot more nodes to combine into + // an offset + let offset_result = self.byte_offset(index, &nodes)?; + + // Construct response of either instructions (Left) or the result (Right) + let mut instructions: Vec = Vec::new(); + let mut byte_range = NodeByteRange { + index: 0, + length: 0, + }; + match length_result { + Either::Left(instruction) => { + if infos.is_some() { + return Err(anyhow!("Could not return size from fetched nodes")); + } + instructions.push(instruction); + } + Either::Right(node) => { + byte_range.length = node.length; + } + } + match offset_result { + Either::Left(offset_instructions) => { + if infos.is_some() { + return Err(anyhow!("Could not return offset from fetched nodes")); + } + instructions.extend(offset_instructions); + } + Either::Right(offset) => { + byte_range.index = offset; + } + } + + if instructions.is_empty() { + Ok(Either::Right(byte_range)) + } else { + Ok(Either::Left(instructions.into_boxed_slice())) + } + } + fn commitable(&self, changeset: &MerkleTreeChangeset) -> bool { let correct_length: bool = if changeset.upgraded { changeset.original_tree_length == self.length @@ -208,6 +287,87 @@ impl MerkleTree { } infos_to_flush } + + fn byte_offset( + &self, + index: u64, + nodes: &Vec, + ) -> Result, u64>> { + let index = if (index & 1) == 1 { + flat_tree::left_span(index) + } else { + index + }; + let mut head: u64 = 0; + let mut offset: u64 = 0; + + for root_node in &self.roots { + head += 2 * ((root_node.index - head) + 1); + + if index >= head { + offset += root_node.length; + continue; + } + let mut iter = flat_tree::Iterator::new(root_node.index); + + let mut instructions: Vec = Vec::new(); + while iter.index() != index { + if index < iter.index() { + iter.left_child(); + } else { + let left_child = iter.left_child(); + let node_or_instruction = self.get_node(left_child, nodes)?; + match node_or_instruction { + Either::Left(instruction) => { + instructions.push(instruction); + } + Either::Right(node) => { + offset += node.length; + } + } + iter.sibling(); + } + } + return if instructions.is_empty() { + Ok(Either::Right(offset)) + } else { + Ok(Either::Left(instructions)) + }; + } + Err(anyhow!( + "Could not calculate byte offset for index {}", + index + )) + } + + fn get_node( + &self, + index: u64, + nodes: &Vec, + ) -> Result> { + // First check if unflushed already has the node + if let Some(node) = self.unflushed.get(index) { + if node.blank || (self.truncated && node.index >= 2 * self.truncate_to) { + // The node is either blank or being deleted + return Err(anyhow!("Could not load node: {}", index)); + } + return Ok(Either::Right(node.clone())); + } + + // Then check if it's already in the incoming nodes + for node in nodes { + if node.index == index { + return Ok(Either::Right(node.clone())); + } + } + + // If not, retunr an instruction + Ok(Either::Left(StoreInfoInstruction::new_content( + Store::Tree, + 40 * index, + 40, + ))) + } } fn get_root_indices(header_tree_length: &u64) -> Vec { @@ -216,6 +376,10 @@ fn get_root_indices(header_tree_length: &u64) -> Vec { roots } +fn index_from_info(info: &StoreInfo) -> u64 { + info.index / NODE_SIZE +} + fn node_from_bytes(index: &u64, data: &[u8]) -> Node { let len_buf = &data[..8]; let hash = &data[8..]; diff --git a/tests/js_interop.rs b/tests/js_interop.rs index 0043e1a8..9941d3ec 100644 --- a/tests/js_interop.rs +++ b/tests/js_interop.rs @@ -78,7 +78,23 @@ async fn step_3_read_and_append_unflushed(work_dir: &str) -> Result<()> { let path = Path::new(work_dir).to_owned(); let key_pair = get_test_key_pair(); let storage = Storage::new_disk(&path, false).await?; - let mut _hypercore = Hypercore::new_with_key_pair(storage, key_pair).await?; + let mut hypercore = Hypercore::new_with_key_pair(storage, key_pair).await?; + let hello = hypercore.get(0).await?; + assert_eq!(hello.unwrap(), b"Hello"); + let world = hypercore.get(1).await?; + assert_eq!(world.unwrap(), b"World"); + let append_outcome = hypercore.append(b"first").await?; + assert_eq!(append_outcome.length, 3); + assert_eq!(append_outcome.byte_length, 15); + let append_outcome = hypercore.append_batch(&[b"second", b"third"]).await?; + assert_eq!(append_outcome.length, 5); + assert_eq!(append_outcome.byte_length, 26); + let first = hypercore.get(2).await?; + assert_eq!(first.unwrap(), b"first"); + let second = hypercore.get(3).await?; + assert_eq!(second.unwrap(), b"second"); + let third = hypercore.get(3).await?; + assert_eq!(third.unwrap(), b"third"); Ok(()) } From 934a608f28c0d513a170b3169faaa8f42db7eeb7 Mon Sep 17 00:00:00 2001 From: Timo Tiuraniemi Date: Mon, 12 Sep 2022 16:27:59 +0300 Subject: [PATCH 053/157] Reading and simple appending without flushing works --- src/core.rs | 74 +++++++++++++++++++++-------------------- src/oplog/mod.rs | 3 +- src/tree/merkle_tree.rs | 5 +++ tests/js/interop.js | 4 +++ tests/js_interop.rs | 5 ++- 5 files changed, 53 insertions(+), 38 deletions(-) diff --git a/src/core.rs b/src/core.rs index d784f8dd..b6f1fff6 100644 --- a/src/core.rs +++ b/src/core.rs @@ -114,46 +114,48 @@ where None => anyhow::bail!("No secret key, cannot append."), }; - // Create a changeset for the tree - let mut changeset = self.tree.changeset(); - let mut batch_length: usize = 0; - for data in batch.iter() { - batch_length += changeset.append(data); - } - changeset.hash_and_sign(&self.key_pair.public, &secret_key); - - // Write the received data to the block store - let info = self - .block_store - .append_batch(batch, batch_length, self.tree.byte_length); - self.storage.flush_info(info).await?; - - // Append the changeset to the Oplog - let outcome = self.oplog.append_changeset(&changeset, false, &self.header); - self.storage.flush_infos(&outcome.infos_to_flush).await?; - self.header = outcome.header; - - // Write to bitfield - self.bitfield.set_range( - changeset.ancestors, - changeset.length - changeset.ancestors, - true, - ); - - // Contiguous length is known only now - self.update_contiguous_length(false, changeset.ancestors, changeset.batch_length); - - // Commit changeset to in-memory tree - self.tree.commit(changeset)?; - - // Now ready to flush - if self.should_flush_bitfield_and_tree_and_oplog() { - self.flush_bitfield_and_tree_and_oplog().await?; + if !batch.is_empty() { + // Create a changeset for the tree + let mut changeset = self.tree.changeset(); + let mut batch_length: usize = 0; + for data in batch.iter() { + batch_length += changeset.append(data); + } + changeset.hash_and_sign(&self.key_pair.public, &secret_key); + + // Write the received data to the block store + let info = self + .block_store + .append_batch(batch, batch_length, self.tree.byte_length); + self.storage.flush_info(info).await?; + + // Append the changeset to the Oplog + let outcome = self.oplog.append_changeset(&changeset, false, &self.header); + self.storage.flush_infos(&outcome.infos_to_flush).await?; + self.header = outcome.header; + + // Write to bitfield + self.bitfield.set_range( + changeset.ancestors, + changeset.length - changeset.ancestors, + true, + ); + + // Contiguous length is known only now + self.update_contiguous_length(false, changeset.ancestors, changeset.batch_length); + + // Commit changeset to in-memory tree + self.tree.commit(changeset)?; + + // Now ready to flush + if self.should_flush_bitfield_and_tree_and_oplog() { + self.flush_bitfield_and_tree_and_oplog().await?; + } } + // Return the new value Ok(AppendOutcome { length: self.tree.length, - // TODO: This comes in JS from the block store write result byte_length: self.tree.byte_length, }) } diff --git a/src/oplog/mod.rs b/src/oplog/mod.rs index 0cc077d4..64eb70f0 100644 --- a/src/oplog/mod.rs +++ b/src/oplog/mod.rs @@ -135,6 +135,7 @@ impl Oplog { .as_ref() .expect("Changeset must be signed before appended"); let signature: Box<[u8]> = signature.to_bytes().into(); + let entry: Entry = Entry { user_data: vec![], tree_nodes, @@ -152,7 +153,7 @@ impl Oplog { }; let mut header: Header = header.clone(); - header.tree.length = changeset.batch_length; + header.tree.length = changeset.length; header.tree.root_hash = hash.clone(); header.tree.signature = signature; diff --git a/src/tree/merkle_tree.rs b/src/tree/merkle_tree.rs index 3d85263a..5d158cb6 100644 --- a/src/tree/merkle_tree.rs +++ b/src/tree/merkle_tree.rs @@ -55,6 +55,7 @@ impl MerkleTree { let mut roots: Vec = Vec::with_capacity(infos.len()); let mut byte_length: u64 = 0; let mut length: u64 = 0; + for i in 0..root_indices.len() { let index = root_indices[i]; ensure!( @@ -69,6 +70,9 @@ impl MerkleTree { roots.push(node); } + if length > 0 { + length = length / 2; + } Ok(Self { roots, @@ -99,6 +103,7 @@ impl MerkleTree { if changeset.upgraded { self.commit_truncation(&changeset); + self.roots = changeset.roots; self.length = changeset.length; self.byte_length = changeset.byte_length; diff --git a/tests/js/interop.js b/tests/js/interop.js index cfaf495a..836ca6a1 100644 --- a/tests/js/interop.js +++ b/tests/js/interop.js @@ -68,6 +68,10 @@ async function step3iReadAndAppendUnflushed(testSet) { if (result.length != 5 || result.byteLength != 26) { throw new Error(`Invalid append result: ${result.length} or ${result.byteLength}`); } + result = await core.append([]); + if (result.length != 5 || result.byteLength != 26) { + throw new Error(`Invalid append result: ${result.length} or ${result.byteLength}`); + } const first = (await core.get(2)).toString(); const second = (await core.get(3)).toString(); const third = (await core.get(4)).toString(); diff --git a/tests/js_interop.rs b/tests/js_interop.rs index 9941d3ec..bc65d68f 100644 --- a/tests/js_interop.rs +++ b/tests/js_interop.rs @@ -89,11 +89,14 @@ async fn step_3_read_and_append_unflushed(work_dir: &str) -> Result<()> { let append_outcome = hypercore.append_batch(&[b"second", b"third"]).await?; assert_eq!(append_outcome.length, 5); assert_eq!(append_outcome.byte_length, 26); + let append_outcome = hypercore.append_batch(&[]).await?; + assert_eq!(append_outcome.length, 5); + assert_eq!(append_outcome.byte_length, 26); let first = hypercore.get(2).await?; assert_eq!(first.unwrap(), b"first"); let second = hypercore.get(3).await?; assert_eq!(second.unwrap(), b"second"); - let third = hypercore.get(3).await?; + let third = hypercore.get(4).await?; assert_eq!(third.unwrap(), b"third"); Ok(()) } From 162ed84ee311a5b7aeb60dac5b74815dd3519447 Mon Sep 17 00:00:00 2001 From: Timo Tiuraniemi Date: Tue, 13 Sep 2022 10:09:24 +0300 Subject: [PATCH 054/157] Setup failing test for two entries to be read from the oplog on open. --- tests/js/interop.js | 59 +++++++++++++++++++++++++++--------------- tests/js_interop.rs | 62 +++++++++++++++++++++++++++++++++------------ 2 files changed, 85 insertions(+), 36 deletions(-) diff --git a/tests/js/interop.js b/tests/js/interop.js index 836ca6a1..4b97a8bd 100644 --- a/tests/js/interop.js +++ b/tests/js/interop.js @@ -31,9 +31,13 @@ if (process.argv[2] === '1') { console.log("step2 ready", result); }); } else if (process.argv[2] === '3'){ - step3iReadAndAppendUnflushed(process.argv[3]).then(result => { + step3ReadAndAppendUnflushed(process.argv[3]).then(result => { console.log("step3 ready", result); }); +} else if (process.argv[2] === '4'){ + step4AppendWithFlush(process.argv[3]).then(result => { + console.log("step4 ready", result); + }); } else { console.error(`Invalid test step {}`, process.argv[2]); process.exit(2); @@ -47,36 +51,51 @@ async function step1Create(testSet) { async function step2AppendHelloWorld(testSet) { const core = new Hypercore(`work/${testSet}`, testKeyPair.publicKey, {keyPair: testKeyPair}); const result = await core.append([Buffer.from('Hello'), Buffer.from('World')]); - if (result.length != 2 || result.byteLength != 10) { - throw new Error(`Invalid append result: ${result.length} or ${result.byteLength}`); - } + assert(result.length, 2); + assert(result.byteLength, 10); await core.close(); }; -async function step3iReadAndAppendUnflushed(testSet) { +async function step3ReadAndAppendUnflushed(testSet) { const core = new Hypercore(`work/${testSet}`, testKeyPair.publicKey, {keyPair: testKeyPair}); const hello = (await core.get(0)).toString(); const world = (await core.get(1)).toString(); - if (hello != "Hello" || world != "World") { - throw new Error(`Read invalid data from hypercore: ${hello} or ${world}`); - } + assert(hello, "Hello"); + assert(world, "World"); let result = await core.append(Buffer.from('first')); - if (result.length != 3 || result.byteLength != 15) { - throw new Error(`Invalid append result: ${result.length} or ${result.byteLength}`); - } + assert(result.length, 3); + assert(result.byteLength, 15); result = await core.append([Buffer.from('second'), Buffer.from('third')]); - if (result.length != 5 || result.byteLength != 26) { - throw new Error(`Invalid append result: ${result.length} or ${result.byteLength}`); - } + assert(result.length, 5); + assert(result.byteLength, 26); + result = await core.append(Buffer.from('fourth')); + assert(result.length, 6); + assert(result.byteLength, 32); result = await core.append([]); - if (result.length != 5 || result.byteLength != 26) { - throw new Error(`Invalid append result: ${result.length} or ${result.byteLength}`); - } + assert(result.length, 6); + assert(result.byteLength, 32); const first = (await core.get(2)).toString(); + assert(first, "first"); const second = (await core.get(3)).toString(); + assert(second, "second"); const third = (await core.get(4)).toString(); - if (first != "first" || second != "second" || third != "third") { - throw new Error(`Read invalid data from hypercore: ${first} or ${second} or ${third}`); - } + assert(third, "third"); + const fourth = (await core.get(5)).toString(); + assert(fourth, "fourth"); await core.close(); }; + +async function step4AppendWithFlush(testSet) { + const core = new Hypercore(`work/${testSet}`, testKeyPair.publicKey, {keyPair: testKeyPair}); + for (let i=0; i<5; i++) { + result = await core.append(Buffer.from([i])); + assert(result.length, 6+i+1); + assert(result.byteLength, 32+i+1); + } +} + +function assert(real, expected) { + if (real != expected) { + throw new Error(`Got ${real} but expected ${expected}`); + } +} diff --git a/tests/js_interop.rs b/tests/js_interop.rs index bc65d68f..b5812da8 100644 --- a/tests/js_interop.rs +++ b/tests/js_interop.rs @@ -7,6 +7,7 @@ use common::{create_hypercore_hash, get_test_key_pair}; #[cfg(feature = "v10")] use hypercore::{Hypercore, Storage}; use js::{cleanup, install, js_run_step, prepare_test_set}; +use random_access_disk::RandomAccessDisk; const TEST_SET_JS_FIRST: &str = "jsfirst"; const TEST_SET_RS_FIRST: &str = "rsfirst"; @@ -33,6 +34,8 @@ async fn js_interop_js_first() -> Result<()> { assert_eq!(create_hypercore_hash(&work_dir), step_2_hash()); js_run_step(3, TEST_SET_JS_FIRST); assert_eq!(create_hypercore_hash(&work_dir), step_3_hash()); + step_4_append_with_flush(&work_dir).await?; + assert_eq!(create_hypercore_hash(&work_dir), step_4_hash()); Ok(()) } @@ -49,24 +52,20 @@ async fn js_interop_rs_first() -> Result<()> { assert_eq!(create_hypercore_hash(&work_dir), step_2_hash()); step_3_read_and_append_unflushed(&work_dir).await?; assert_eq!(create_hypercore_hash(&work_dir), step_3_hash()); + js_run_step(4, TEST_SET_RS_FIRST); + assert_eq!(create_hypercore_hash(&work_dir), step_4_hash()); Ok(()) } #[cfg(feature = "v10")] async fn step_1_create(work_dir: &str) -> Result<()> { - let path = Path::new(work_dir).to_owned(); - let key_pair = get_test_key_pair(); - let storage = Storage::new_disk(&path, false).await?; - Hypercore::new_with_key_pair(storage, key_pair).await?; + get_hypercore(work_dir).await?; Ok(()) } #[cfg(feature = "v10")] async fn step_2_append_hello_world(work_dir: &str) -> Result<()> { - let path = Path::new(work_dir).to_owned(); - let key_pair = get_test_key_pair(); - let storage = Storage::new_disk(&path, false).await?; - let mut hypercore = Hypercore::new_with_key_pair(storage, key_pair).await?; + let mut hypercore = get_hypercore(work_dir).await?; let append_outcome = hypercore.append_batch(&[b"Hello", b"World"]).await?; assert_eq!(append_outcome.length, 2); assert_eq!(append_outcome.byte_length, 10); @@ -75,10 +74,7 @@ async fn step_2_append_hello_world(work_dir: &str) -> Result<()> { #[cfg(feature = "v10")] async fn step_3_read_and_append_unflushed(work_dir: &str) -> Result<()> { - let path = Path::new(work_dir).to_owned(); - let key_pair = get_test_key_pair(); - let storage = Storage::new_disk(&path, false).await?; - let mut hypercore = Hypercore::new_with_key_pair(storage, key_pair).await?; + let mut hypercore = get_hypercore(work_dir).await?; let hello = hypercore.get(0).await?; assert_eq!(hello.unwrap(), b"Hello"); let world = hypercore.get(1).await?; @@ -89,18 +85,43 @@ async fn step_3_read_and_append_unflushed(work_dir: &str) -> Result<()> { let append_outcome = hypercore.append_batch(&[b"second", b"third"]).await?; assert_eq!(append_outcome.length, 5); assert_eq!(append_outcome.byte_length, 26); + let append_outcome = hypercore.append(b"fourth").await?; + assert_eq!(append_outcome.length, 6); + assert_eq!(append_outcome.byte_length, 32); let append_outcome = hypercore.append_batch(&[]).await?; - assert_eq!(append_outcome.length, 5); - assert_eq!(append_outcome.byte_length, 26); + assert_eq!(append_outcome.length, 6); + assert_eq!(append_outcome.byte_length, 32); let first = hypercore.get(2).await?; assert_eq!(first.unwrap(), b"first"); let second = hypercore.get(3).await?; assert_eq!(second.unwrap(), b"second"); let third = hypercore.get(4).await?; assert_eq!(third.unwrap(), b"third"); + let fourth = hypercore.get(5).await?; + assert_eq!(fourth.unwrap(), b"fourth"); + Ok(()) +} + +#[cfg(feature = "v10")] +async fn step_4_append_with_flush(work_dir: &str) -> Result<()> { + let mut hypercore = get_hypercore(work_dir).await?; + for i in 0..5 { + let append_outcome = hypercore.append(&[i]).await?; + println!("GOT APPEND {:?}", append_outcome); + assert_eq!(append_outcome.length, (5 + i + 1) as u64); + assert_eq!(append_outcome.byte_length, (26 + i + 1) as u64); + } Ok(()) } +#[cfg(feature = "v10")] +async fn get_hypercore(work_dir: &str) -> Result> { + let path = Path::new(work_dir).to_owned(); + let key_pair = get_test_key_pair(); + let storage = Storage::new_disk(&path, false).await?; + Ok(Hypercore::new_with_key_pair(storage, key_pair).await?) +} + fn step_0_hash() -> common::HypercoreHash { common::HypercoreHash { bitfield: None, @@ -131,8 +152,17 @@ fn step_2_hash() -> common::HypercoreHash { fn step_3_hash() -> common::HypercoreHash { common::HypercoreHash { bitfield: Some("DEC1593A7456C8C9407B9B8B9C89682DFFF33C3892BCC9D9F06956FEE0A1B949".into()), - data: Some("402670849413BB97FAC3A322FC1EE3DE20F7F0D9C641B0F70BC4C619B3032C50".into()), - oplog: Some("0536819D13798DADFCA9A8607D10DDD14254902FBCC35E97043039E4308869B5".into()), + data: Some("A9C34DA27BF72075C2435F8D4EE2DEC75F7AD1ADB31CE4782AFBBC0C6FDEDF1F".into()), + oplog: Some("94E4E7CFB873212B7A38EFAEC4B0BB5426741793ADB9EFC15484C0CEBBD6012B".into()), tree: Some("38788609A8634DC8D34F9AE723F3169ADB20768ACFDFF266A43B7E217750DD1E".into()), } } + +fn step_4_hash() -> common::HypercoreHash { + common::HypercoreHash { + bitfield: Some("9B844E9378A7D13D6CDD4C1FF12FB313013E5CC472C6CB46497033563FE6B8F1".into()), + data: Some("ADB6D70826037B3E24EB7A9D8BEE314B6B4596812E5FE9C737EB883CB584EDC2".into()), + oplog: Some("39B68580C64A0F96599011E25C539B9F7F89276586DCF12A1F0B1C6446F0D024".into()), + tree: Some("4F346485415AE9A068490764F85CA6307E351C0C8DBD4192F16A9608F5D6F339".into()), + } +} From bbcc2d9e3b3e6363d5093fe7d47b88dd665e3981 Mon Sep 17 00:00:00 2001 From: Timo Tiuraniemi Date: Tue, 13 Sep 2022 14:06:39 +0300 Subject: [PATCH 055/157] Implemented oplog entries read on open, and flushing of changes at the same time as JS --- src/core.rs | 125 ++++++++++++++++++++++-------- src/oplog/mod.rs | 66 ++++++++++++---- src/tree/merkle_tree.rs | 77 +++++++++++++++--- src/tree/merkle_tree_changeset.rs | 9 ++- tests/js_interop.rs | 5 +- 5 files changed, 218 insertions(+), 64 deletions(-) diff --git a/src/core.rs b/src/core.rs index b6f1fff6..c9a06b57 100644 --- a/src/core.rs +++ b/src/core.rs @@ -10,6 +10,7 @@ use crate::{ tree::MerkleTree, }; use anyhow::{anyhow, Result}; +use ed25519_dalek::Signature; use futures::future::Either; use random_access_storage::RandomAccess; use std::fmt::Debug; @@ -66,7 +67,7 @@ where .data .expect("Did not receive data"); - let oplog_open_outcome = Oplog::open(key_pair.clone(), oplog_bytes)?; + let mut oplog_open_outcome = Oplog::open(key_pair.clone(), oplog_bytes)?; storage .flush_infos(&oplog_open_outcome.infos_to_flush) .await?; @@ -75,7 +76,7 @@ where let info_instructions = MerkleTree::get_info_instructions_to_read(&oplog_open_outcome.header.tree); let infos = storage.read_infos(&info_instructions).await?; - let tree = MerkleTree::open(&oplog_open_outcome.header.tree, infos)?; + let mut tree = MerkleTree::open(&oplog_open_outcome.header.tree, infos)?; // Create block store instance let block_store = BlockStore::default(); @@ -88,7 +89,59 @@ where .expect("Did not get store length with size instruction"); let info_instruction = Bitfield::get_info_instruction_to_read(bitfield_store_length); let info = storage.read_info(info_instruction).await?; - let bitfield = Bitfield::open(info); + let mut bitfield = Bitfield::open(info); + + // Process entries stored only to the oplog and not yet flushed into bitfield or tree + if let Some(entries) = oplog_open_outcome.entries { + for entry in entries.iter() { + for node in &entry.tree_nodes { + tree.add_node(node.clone()); + } + + if let Some(entry_bitfield) = &entry.bitfield { + bitfield.set_range( + entry_bitfield.start, + entry_bitfield.length, + !entry_bitfield.drop, + ); + update_contiguous_length( + &mut oplog_open_outcome.header, + &bitfield, + entry_bitfield.drop, + entry_bitfield.start, + entry_bitfield.length, + ); + } + if let Some(tree_upgrade) = &entry.tree_upgrade { + // TODO: Generalize Either response stack + let mut changeset = + match tree.truncate(tree_upgrade.length, tree_upgrade.fork, None)? { + Either::Right(changeset) => changeset, + Either::Left(instructions) => { + let infos = storage.read_infos(&instructions).await?; + match tree.truncate( + tree_upgrade.length, + tree_upgrade.fork, + Some(&infos), + )? { + Either::Right(changeset) => changeset, + Either::Left(_) => { + return Err(anyhow!("Could not truncate")); + } + } + } + }; + changeset.ancestors = tree_upgrade.ancestors; + changeset.signature = Some(Signature::from_bytes(&tree_upgrade.signature)?); + + // TODO: Skip reorg hints for now, seems to only have to do with replication + // addReorgHint(header.hints.reorgs, tree, batch) + + // Commit changeset to in-memory tree + tree.commit(changeset)?; + } + } + } Ok(Hypercore { key_pair, @@ -142,7 +195,13 @@ where ); // Contiguous length is known only now - self.update_contiguous_length(false, changeset.ancestors, changeset.batch_length); + update_contiguous_length( + &mut self.header, + &self.bitfield, + false, + changeset.ancestors, + changeset.batch_length, + ); // Commit changeset to in-memory tree self.tree.commit(changeset)?; @@ -166,7 +225,7 @@ where return Ok(None); } - // TODO: Figure out a way to generalize this Either processing stack! + // TODO: Generalize Either response stack let byte_range = match self.tree.byte_range(index, None)? { Either::Right(byte_range) => byte_range, Either::Left(instructions) => { @@ -180,6 +239,7 @@ where } }; + // TODO: Generalize Either response stack let data = match self.block_store.read(&byte_range, None) { Either::Right(data) => data, Either::Left(instruction) => { @@ -200,7 +260,7 @@ where if self.skip_flush_count == 0 || self.oplog.entries_byte_length >= MAX_OPLOG_ENTRIES_BYTE_SIZE { - self.skip_flush_count = 4; + self.skip_flush_count = 3; true } else { self.skip_flush_count -= 1; @@ -208,32 +268,6 @@ where } } - fn update_contiguous_length( - &mut self, - bitfield_drop: bool, - bitfield_start: u64, - bitfield_length: u64, - ) { - let end = bitfield_start + bitfield_length; - let mut c = self.header.contiguous_length; - if bitfield_drop { - if c <= end && c > bitfield_start { - c = bitfield_start; - } - } else { - if c <= end && c >= bitfield_start { - c = end; - while self.bitfield.get(c) { - c += 1; - } - } - } - - if c != self.header.contiguous_length { - self.header.contiguous_length = c; - } - } - async fn flush_bitfield_and_tree_and_oplog(&mut self) -> Result<()> { let infos = self.bitfield.flush(); self.storage.flush_infos(&infos).await?; @@ -244,3 +278,30 @@ where Ok(()) } } + +fn update_contiguous_length( + header: &mut Header, + bitfield: &Bitfield, + bitfield_drop: bool, + bitfield_start: u64, + bitfield_length: u64, +) { + let end = bitfield_start + bitfield_length; + let mut c = header.contiguous_length; + if bitfield_drop { + if c <= end && c > bitfield_start { + c = bitfield_start; + } + } else { + if c <= end && c >= bitfield_start { + c = end; + while bitfield.get(c) { + c += 1; + } + } + } + + if c != header.contiguous_length { + header.contiguous_length = c; + } +} diff --git a/src/oplog/mod.rs b/src/oplog/mod.rs index 64eb70f0..88b89bdb 100644 --- a/src/oplog/mod.rs +++ b/src/oplog/mod.rs @@ -38,14 +38,27 @@ pub struct OplogOpenOutcome { pub oplog: Oplog, pub header: Header, pub infos_to_flush: Box<[StoreInfo]>, + pub entries: Option>, } impl OplogOpenOutcome { - pub fn new(oplog: Oplog, create_header_outcome: OplogCreateHeaderOutcome) -> Self { + pub fn new(oplog: Oplog, header: Header, infos_to_flush: Box<[StoreInfo]>) -> Self { + Self { + oplog, + header, + infos_to_flush, + entries: None, + } + } + pub fn from_create_header_outcome( + oplog: Oplog, + create_header_outcome: OplogCreateHeaderOutcome, + ) -> Self { Self { oplog, header: create_header_outcome.header, infos_to_flush: create_header_outcome.infos_to_flush, + entries: None, } } } @@ -76,7 +89,7 @@ impl Oplog { // Depending on what is stored, the state needs to be set accordingly. // See `get_next_header_oplog_slot_and_bit_value` for details on header_bits. - if let Some(mut h1_outcome) = h1_outcome { + let mut outcome: OplogOpenOutcome = if let Some(mut h1_outcome) = h1_outcome { let (header, header_bits): (Header, [bool; 2]) = if let Some(mut h2_outcome) = h2_outcome { let header_bits = [h1_outcome.header_bit, h2_outcome.header_bit]; @@ -97,11 +110,7 @@ impl Oplog { entries_length: 0, entries_byte_length: 0, }; - Ok(OplogOpenOutcome { - oplog, - header, - infos_to_flush: Box::new([]), - }) + OplogOpenOutcome::new(oplog, header, Box::new([])) } else if let Some(mut h2_outcome) = h2_outcome { // This shouldn't happen because the first header is saved to the first slot // but Javascript supports this so we should too. @@ -111,15 +120,37 @@ impl Oplog { entries_length: 0, entries_byte_length: 0, }; - Ok(OplogOpenOutcome { - oplog, - header: h2_outcome.state.decode(&existing), - infos_to_flush: Box::new([]), - }) + OplogOpenOutcome::new(oplog, h2_outcome.state.decode(&existing), Box::new([])) } else { // There is nothing in the oplog, start from new. - Ok(Self::new(key_pair)) + Self::new(key_pair) + }; + + // Read headers that might be stored in the existing content + if existing.len() > OplogSlot::Entries as usize { + let mut entry_offset = OplogSlot::Entries as usize; + let mut entries: Vec = Vec::new(); + let mut partials: Vec = Vec::new(); + loop { + if let Some(mut entry_outcome) = + Self::validate_leader(entry_offset as usize, &existing)? + { + let entry: Entry = entry_outcome.state.decode(&existing); + entries.push(entry); + partials.push(entry_outcome.partial_bit); + entry_offset = entry_outcome.state.end; + } else { + break; + } + } + + // Remove all trailing partial entries + while partials.len() > 0 && partials[partials.len() - 1] { + entries.pop(); + } + outcome.entries = Some(entries.into_boxed_slice()); } + Ok(outcome) } /// Appends a changeset to the Oplog. @@ -130,9 +161,12 @@ impl Oplog { header: &Header, ) -> OplogCreateHeaderOutcome { let tree_nodes: Vec = changeset.nodes.clone(); - let (hash, signature) = changeset - .hash_and_signature + let hash = changeset + .hash .as_ref() + .expect("Changeset must have a hash before appended"); + let signature = changeset + .signature .expect("Changeset must be signed before appended"); let signature: Box<[u8]> = signature.to_bytes().into(); @@ -217,7 +251,7 @@ impl Oplog { entries_length, entries_byte_length, }; - OplogOpenOutcome::new( + OplogOpenOutcome::from_create_header_outcome( oplog, OplogCreateHeaderOutcome { header, diff --git a/src/tree/merkle_tree.rs b/src/tree/merkle_tree.rs index 5d158cb6..5f05c7e4 100644 --- a/src/tree/merkle_tree.rs +++ b/src/tree/merkle_tree.rs @@ -108,9 +108,7 @@ impl MerkleTree { self.length = changeset.length; self.byte_length = changeset.byte_length; self.fork = changeset.fork; - self.signature = changeset - .hash_and_signature - .map(|hash_and_signature| hash_and_signature.1); + self.signature = changeset.signature; } for node in changeset.nodes { @@ -154,13 +152,7 @@ impl MerkleTree { } // Get nodes out of incoming infos - let nodes: Vec = match infos { - Some(infos) => infos - .iter() - .map(|info| node_from_bytes(&index_from_info(&info), info.data.as_ref().unwrap())) - .collect(), - None => vec![], - }; + let nodes: Vec = infos_to_nodes(infos); // Start with getting the requested node, which will get the length // of the byte range @@ -206,6 +198,61 @@ impl MerkleTree { } } + pub fn add_node(&mut self, node: Node) { + self.unflushed.insert(node.index, node); + } + + pub fn truncate( + &mut self, + length: u64, + fork: u64, + infos: Option<&[StoreInfo]>, + ) -> Result, MerkleTreeChangeset>> { + let head = length * 2; + let mut full_roots = vec![]; + flat_tree::full_roots(head, &mut full_roots); + let nodes: Vec = infos_to_nodes(infos); + let mut changeset = self.changeset(); + + let mut instructions: Vec = Vec::new(); + for i in 0..full_roots.len() { + let root = full_roots[i]; + if i < changeset.roots.len() && changeset.roots[i].index == root { + continue; + } + while changeset.roots.len() > i { + changeset.roots.pop(); + } + + let node_or_instruction = self.get_node(root, &nodes)?; + match node_or_instruction { + Either::Left(instruction) => { + instructions.push(instruction); + } + Either::Right(node) => { + changeset.roots.push(node); + } + } + } + + if instructions.is_empty() { + while changeset.roots.len() > full_roots.len() { + changeset.roots.pop(); + } + changeset.fork = fork; + changeset.length = length; + changeset.ancestors = length; + changeset.byte_length = changeset + .roots + .iter() + .fold(0, |acc, node| acc + node.length); + changeset.upgraded = true; + Ok(Either::Right(changeset)) + } else { + Ok(Either::Left(instructions.into_boxed_slice())) + } + } + fn commitable(&self, changeset: &MerkleTreeChangeset) -> bool { let correct_length: bool = if changeset.upgraded { changeset.original_tree_length == self.length @@ -392,3 +439,13 @@ fn node_from_bytes(index: &u64, data: &[u8]) -> Node { let len = state.decode_u64(len_buf); Node::new(*index, hash.to_vec(), len) } + +fn infos_to_nodes(infos: Option<&[StoreInfo]>) -> Vec { + match infos { + Some(infos) => infos + .iter() + .map(|info| node_from_bytes(&index_from_info(&info), info.data.as_ref().unwrap())) + .collect(), + None => vec![], + } +} diff --git a/src/tree/merkle_tree_changeset.rs b/src/tree/merkle_tree_changeset.rs index 1e782808..2c18fefb 100644 --- a/src/tree/merkle_tree_changeset.rs +++ b/src/tree/merkle_tree_changeset.rs @@ -20,7 +20,8 @@ pub struct MerkleTreeChangeset { pub(crate) fork: u64, pub(crate) roots: Vec, pub(crate) nodes: Vec, - pub(crate) hash_and_signature: Option<(Box<[u8]>, Signature)>, + pub(crate) hash: Option>, + pub(crate) signature: Option, pub(crate) upgraded: bool, // Safeguarding values @@ -38,7 +39,8 @@ impl MerkleTreeChangeset { fork, roots, nodes: vec![], - hash_and_signature: None, + hash: None, + signature: None, upgraded: false, original_tree_length: length, original_tree_fork: fork, @@ -87,7 +89,8 @@ impl MerkleTreeChangeset { let hash = self.hash(); let signable = signable_tree(&hash, self.length, self.fork); let signature = sign(&public_key, &secret_key, &signable); - self.hash_and_signature = Some((hash, signature)); + self.hash = Some(hash); + self.signature = Some(signature); } /// Calculates a hash of the current set of roots diff --git a/tests/js_interop.rs b/tests/js_interop.rs index b5812da8..8e99b90a 100644 --- a/tests/js_interop.rs +++ b/tests/js_interop.rs @@ -107,9 +107,8 @@ async fn step_4_append_with_flush(work_dir: &str) -> Result<()> { let mut hypercore = get_hypercore(work_dir).await?; for i in 0..5 { let append_outcome = hypercore.append(&[i]).await?; - println!("GOT APPEND {:?}", append_outcome); - assert_eq!(append_outcome.length, (5 + i + 1) as u64); - assert_eq!(append_outcome.byte_length, (26 + i + 1) as u64); + assert_eq!(append_outcome.length, (6 + i + 1) as u64); + assert_eq!(append_outcome.byte_length, (32 + i + 1) as u64); } Ok(()) } From 2dcb22f11832258f6719ceb99848bcd3f0672a7c Mon Sep 17 00:00:00 2001 From: Timo Tiuraniemi Date: Wed, 14 Sep 2022 09:10:38 +0300 Subject: [PATCH 056/157] Use the same Either return value everywhere --- src/bitfield_v10/dynamic.rs | 83 ++++++++++++++++----------- src/common/store.rs | 2 +- src/core.rs | 54 ++++++++++++------ src/tree/merkle_tree.rs | 109 +++++++++++++++++++----------------- 4 files changed, 144 insertions(+), 104 deletions(-) diff --git a/src/bitfield_v10/dynamic.rs b/src/bitfield_v10/dynamic.rs index f4179354..1405fbd2 100644 --- a/src/bitfield_v10/dynamic.rs +++ b/src/bitfield_v10/dynamic.rs @@ -1,8 +1,9 @@ -use super::fixed::{FixedBitfield, FIXED_BITFIELD_BYTES_LENGTH, FIXED_BITFIELD_LENGTH}; +use super::fixed::{FixedBitfield, FIXED_BITFIELD_LENGTH}; use crate::{ - common::{StoreInfo, StoreInfoInstruction}, + common::{StoreInfo, StoreInfoInstruction, StoreInfoType}, Store, }; +use futures::future::Either; use std::{cell::RefCell, convert::TryInto}; const DYNAMIC_BITFIELD_PAGE_SIZE: usize = 32768; @@ -18,36 +19,43 @@ pub struct DynamicBitfield { } impl DynamicBitfield { - /// Gets info instruction to read based on the bitfield store length - pub fn get_info_instruction_to_read(bitfield_store_length: u64) -> StoreInfoInstruction { - // Read only multiples of 4 bytes. Javascript: - // const size = st.size - (st.size & 3) - let length = bitfield_store_length - (bitfield_store_length & 3); - StoreInfoInstruction::new_content(Store::Bitfield, 0, length) - } - - pub fn open(info: StoreInfo) -> Self { - let data = info.data.unwrap(); - let resumed = data.len() >= 4; - if resumed { - let mut pages: intmap::IntMap> = intmap::IntMap::new(); - let mut data_index = 0; - while data_index < data.len() { - let parent_index: u64 = (data_index / FIXED_BITFIELD_LENGTH) as u64; - pages.insert( - parent_index, - RefCell::new(FixedBitfield::from_data(parent_index, data_index, &data)), - ); - data_index += FIXED_BITFIELD_LENGTH; - } - Self { - pages, - unflushed: vec![], - } - } else { - Self { - pages: intmap::IntMap::new(), - unflushed: vec![], + pub fn open(info: Option) -> Either { + match info { + None => Either::Left(StoreInfoInstruction::new_size(Store::Bitfield, 0)), + Some(info) => { + if info.info_type == StoreInfoType::Size { + let bitfield_store_length = info.length.unwrap(); + // Read only multiples of 4 bytes. + let length = bitfield_store_length - (bitfield_store_length & 3); + return Either::Left(StoreInfoInstruction::new_content( + Store::Bitfield, + 0, + length, + )); + } + let data = info.data.expect("Did not receive bitfield store content"); + let resumed = data.len() >= 4; + if resumed { + let mut pages: intmap::IntMap> = intmap::IntMap::new(); + let mut data_index = 0; + while data_index < data.len() { + let parent_index: u64 = (data_index / FIXED_BITFIELD_LENGTH) as u64; + pages.insert( + parent_index, + RefCell::new(FixedBitfield::from_data(parent_index, data_index, &data)), + ); + data_index += FIXED_BITFIELD_LENGTH; + } + Either::Right(Self { + pages, + unflushed: vec![], + }) + } else { + Either::Right(Self { + pages: intmap::IntMap::new(), + unflushed: vec![], + }) + } } } } @@ -142,9 +150,16 @@ mod tests { } } + fn get_dynamic_bitfield() -> DynamicBitfield { + match DynamicBitfield::open(Some(StoreInfo::new_content(Store::Bitfield, 0, &[]))) { + Either::Left(_) => panic!("Could not open bitfield"), + Either::Right(bitfield) => bitfield, + } + } + #[test] fn bitfield_dynamic_get_and_set() { - let mut bitfield = DynamicBitfield::open(StoreInfo::new_content(Store::Bitfield, 0, &[])); + let mut bitfield = get_dynamic_bitfield(); assert_value_range(&bitfield, 0, 9, false); bitfield.set(0, true); assert_eq!(bitfield.get(0), true); @@ -178,7 +193,7 @@ mod tests { #[test] fn bitfield_dynamic_set_range() { - let mut bitfield = DynamicBitfield::open(StoreInfo::new_content(Store::Bitfield, 0, &[])); + let mut bitfield = get_dynamic_bitfield(); bitfield.set_range(0, 2, true); assert_value_range(&bitfield, 0, 2, true); assert_value_range(&bitfield, 3, 61, false); diff --git a/src/common/store.rs b/src/common/store.rs index 50dd7d71..ef9c900b 100644 --- a/src/common/store.rs +++ b/src/common/store.rs @@ -12,7 +12,7 @@ pub enum Store { } /// Information type about a store. -#[derive(Debug)] +#[derive(Debug, PartialEq)] pub enum StoreInfoType { /// Read/write content of the store Content, diff --git a/src/core.rs b/src/core.rs index c9a06b57..ae369cf9 100644 --- a/src/core.rs +++ b/src/core.rs @@ -73,23 +73,41 @@ where .await?; // Open/create tree - let info_instructions = - MerkleTree::get_info_instructions_to_read(&oplog_open_outcome.header.tree); - let infos = storage.read_infos(&info_instructions).await?; - let mut tree = MerkleTree::open(&oplog_open_outcome.header.tree, infos)?; + let mut tree = match MerkleTree::open(&oplog_open_outcome.header.tree, None)? { + Either::Right(value) => value, + Either::Left(instructions) => { + let infos = storage.read_infos(&instructions).await?; + match MerkleTree::open(&oplog_open_outcome.header.tree, Some(&infos))? { + Either::Right(value) => value, + Either::Left(_) => { + return Err(anyhow!("Could not open tree")); + } + } + } + }; // Create block store instance let block_store = BlockStore::default(); // Open bitfield - let bitfield_store_length = storage - .read_info(StoreInfoInstruction::new_size(Store::Bitfield, 0)) - .await? - .length - .expect("Did not get store length with size instruction"); - let info_instruction = Bitfield::get_info_instruction_to_read(bitfield_store_length); - let info = storage.read_info(info_instruction).await?; - let mut bitfield = Bitfield::open(info); + let mut bitfield = match Bitfield::open(None) { + Either::Right(value) => value, + Either::Left(instruction) => { + let info = storage.read_info(instruction).await?; + match Bitfield::open(Some(info)) { + Either::Right(value) => value, + Either::Left(instruction) => { + let info = storage.read_info(instruction).await?; + match Bitfield::open(Some(info)) { + Either::Right(value) => value, + Either::Left(_) => { + return Err(anyhow!("Could not open bitfield")); + } + } + } + } + } + }; // Process entries stored only to the oplog and not yet flushed into bitfield or tree if let Some(entries) = oplog_open_outcome.entries { @@ -116,7 +134,7 @@ where // TODO: Generalize Either response stack let mut changeset = match tree.truncate(tree_upgrade.length, tree_upgrade.fork, None)? { - Either::Right(changeset) => changeset, + Either::Right(value) => value, Either::Left(instructions) => { let infos = storage.read_infos(&instructions).await?; match tree.truncate( @@ -124,7 +142,7 @@ where tree_upgrade.fork, Some(&infos), )? { - Either::Right(changeset) => changeset, + Either::Right(value) => value, Either::Left(_) => { return Err(anyhow!("Could not truncate")); } @@ -227,11 +245,11 @@ where // TODO: Generalize Either response stack let byte_range = match self.tree.byte_range(index, None)? { - Either::Right(byte_range) => byte_range, + Either::Right(value) => value, Either::Left(instructions) => { let infos = self.storage.read_infos(&instructions).await?; match self.tree.byte_range(index, Some(&infos))? { - Either::Right(byte_range) => byte_range, + Either::Right(value) => value, Either::Left(_) => { return Err(anyhow!("Could not read byte range")); } @@ -241,11 +259,11 @@ where // TODO: Generalize Either response stack let data = match self.block_store.read(&byte_range, None) { - Either::Right(data) => data, + Either::Right(value) => value, Either::Left(instruction) => { let info = self.storage.read_info(instruction).await?; match self.block_store.read(&byte_range, Some(info)) { - Either::Right(data) => data, + Either::Right(value) => value, Either::Left(_) => { return Err(anyhow!("Could not read block storage range")); } diff --git a/src/tree/merkle_tree.rs b/src/tree/merkle_tree.rs index 5f05c7e4..518c6091 100644 --- a/src/tree/merkle_tree.rs +++ b/src/tree/merkle_tree.rs @@ -31,59 +31,66 @@ pub struct MerkleTree { const NODE_SIZE: u64 = 40; impl MerkleTree { - /// Gets instructions to slices that should be read from storage - /// based on `HeaderTree` `length` field. Call this before - /// calling `open`. - pub fn get_info_instructions_to_read(header_tree: &HeaderTree) -> Box<[StoreInfoInstruction]> { - let root_indices = get_root_indices(&header_tree.length); - - root_indices - .iter() - .map(|&index| { - StoreInfoInstruction::new_content(Store::Tree, NODE_SIZE * index, NODE_SIZE) - }) - .collect::>() - .into_boxed_slice() - } + /// Opens MerkleTree, based on read infos. + pub fn open( + header_tree: &HeaderTree, + infos: Option<&[StoreInfo]>, + ) -> Result, Self>> { + match infos { + None => { + let root_indices = get_root_indices(&header_tree.length); + + Ok(Either::Left( + root_indices + .iter() + .map(|&index| { + StoreInfoInstruction::new_content( + Store::Tree, + NODE_SIZE * index, + NODE_SIZE, + ) + }) + .collect::>() + .into_boxed_slice(), + )) + } + Some(infos) => { + let root_indices = get_root_indices(&header_tree.length); + + let mut roots: Vec = Vec::with_capacity(infos.len()); + let mut byte_length: u64 = 0; + let mut length: u64 = 0; + + for i in 0..root_indices.len() { + let index = root_indices[i]; + ensure!( + index == index_from_info(&infos[i]), + "Given slices vector not in the correct order" + ); + let data = infos[i].data.as_ref().unwrap(); + let node = node_from_bytes(&index, data); + byte_length += node.length; + // This is totalSpan in Javascript + length += 2 * ((node.index - length) + 1); + + roots.push(node); + } + if length > 0 { + length = length / 2; + } - /// Opens MerkleTree, based on read infos. Call `get_info_instructions_to_read` - /// before calling this to find out which infos to read. The given infos - /// need to be in the same order as the instructions from `get_info_instructions_to_read`! - pub fn open(header_tree: &HeaderTree, infos: Box<[StoreInfo]>) -> Result { - let root_indices = get_root_indices(&header_tree.length); - - let mut roots: Vec = Vec::with_capacity(infos.len()); - let mut byte_length: u64 = 0; - let mut length: u64 = 0; - - for i in 0..root_indices.len() { - let index = root_indices[i]; - ensure!( - index == index_from_info(&infos[i]), - "Given slices vector not in the correct order" - ); - let data = infos[i].data.as_ref().unwrap(); - let node = node_from_bytes(&index, data); - byte_length += node.length; - // This is totalSpan in Javascript - length += 2 * ((node.index - length) + 1); - - roots.push(node); - } - if length > 0 { - length = length / 2; + Ok(Either::Right(Self { + roots, + length, + byte_length, + fork: header_tree.fork, + unflushed: intmap::IntMap::new(), + truncated: false, + truncate_to: 0, + signature: None, + })) + } } - - Ok(Self { - roots, - length, - byte_length, - fork: header_tree.fork, - unflushed: intmap::IntMap::new(), - truncated: false, - truncate_to: 0, - signature: None, - }) } /// Initialize a changeset for this tree. From cba5e1a60272c53fecd175aca84e55a1ea5cafa5 Mon Sep 17 00:00:00 2001 From: Timo Tiuraniemi Date: Wed, 14 Sep 2022 09:28:46 +0300 Subject: [PATCH 057/157] One more into Either return --- src/core.rs | 20 ++++--- src/oplog/mod.rs | 139 ++++++++++++++++++++++++++--------------------- 2 files changed, 88 insertions(+), 71 deletions(-) diff --git a/src/core.rs b/src/core.rs index ae369cf9..912dec48 100644 --- a/src/core.rs +++ b/src/core.rs @@ -2,7 +2,6 @@ use crate::{ bitfield_v10::Bitfield, - common::{Store, StoreInfoInstruction}, crypto::generate_keypair, data::BlockStore, oplog::{Header, Oplog, MAX_OPLOG_ENTRIES_BYTE_SIZE}, @@ -61,13 +60,18 @@ where key_pair: PartialKeypair, ) -> Result> { // Open/create oplog - let oplog_bytes = storage - .read_info(StoreInfoInstruction::new_all_content(Store::Oplog)) - .await? - .data - .expect("Did not receive data"); - - let mut oplog_open_outcome = Oplog::open(key_pair.clone(), oplog_bytes)?; + let mut oplog_open_outcome = match Oplog::open(&key_pair, None)? { + Either::Right(value) => value, + Either::Left(instruction) => { + let info = storage.read_info(instruction).await?; + match Oplog::open(&key_pair, Some(info))? { + Either::Right(value) => value, + Either::Left(_) => { + return Err(anyhow!("Could not open tree")); + } + } + } + }; storage .flush_infos(&oplog_open_outcome.infos_to_flush) .await?; diff --git a/src/oplog/mod.rs b/src/oplog/mod.rs index 88b89bdb..03f96876 100644 --- a/src/oplog/mod.rs +++ b/src/oplog/mod.rs @@ -1,8 +1,9 @@ -use crate::common::{Store, StoreInfo}; +use crate::common::{Store, StoreInfo, StoreInfoInstruction}; use crate::compact_encoding::{CompactEncoding, State}; use crate::tree::MerkleTreeChangeset; use crate::{Node, PartialKeypair}; use anyhow::{anyhow, Result}; +use futures::future::Either; use std::convert::{TryFrom, TryInto}; mod entry; @@ -82,75 +83,87 @@ const INITIAL_HEADER_BITS: [bool; 2] = [true, false]; impl Oplog { /// Opens an existing Oplog from existing byte buffer or creates a new one. - pub fn open(key_pair: PartialKeypair, existing: Box<[u8]>) -> Result { - // First read and validate both headers stored in the existing oplog - let h1_outcome = Self::validate_leader(OplogSlot::FirstHeader as usize, &existing)?; - let h2_outcome = Self::validate_leader(OplogSlot::SecondHeader as usize, &existing)?; - - // Depending on what is stored, the state needs to be set accordingly. - // See `get_next_header_oplog_slot_and_bit_value` for details on header_bits. - let mut outcome: OplogOpenOutcome = if let Some(mut h1_outcome) = h1_outcome { - let (header, header_bits): (Header, [bool; 2]) = - if let Some(mut h2_outcome) = h2_outcome { - let header_bits = [h1_outcome.header_bit, h2_outcome.header_bit]; - let header: Header = if header_bits[0] == header_bits[1] { - h1_outcome.state.decode(&existing) - } else { - h2_outcome.state.decode(&existing) + pub fn open( + key_pair: &PartialKeypair, + info: Option, + ) -> Result> { + match info { + None => Ok(Either::Left(StoreInfoInstruction::new_all_content( + Store::Oplog, + ))), + Some(info) => { + let existing = info.data.expect("Could not get data of existing oplog"); + // First read and validate both headers stored in the existing oplog + let h1_outcome = Self::validate_leader(OplogSlot::FirstHeader as usize, &existing)?; + let h2_outcome = + Self::validate_leader(OplogSlot::SecondHeader as usize, &existing)?; + + // Depending on what is stored, the state needs to be set accordingly. + // See `get_next_header_oplog_slot_and_bit_value` for details on header_bits. + let mut outcome: OplogOpenOutcome = if let Some(mut h1_outcome) = h1_outcome { + let (header, header_bits): (Header, [bool; 2]) = + if let Some(mut h2_outcome) = h2_outcome { + let header_bits = [h1_outcome.header_bit, h2_outcome.header_bit]; + let header: Header = if header_bits[0] == header_bits[1] { + h1_outcome.state.decode(&existing) + } else { + h2_outcome.state.decode(&existing) + }; + (header, header_bits) + } else { + ( + h1_outcome.state.decode(&existing), + [h1_outcome.header_bit, h1_outcome.header_bit], + ) + }; + let oplog = Oplog { + header_bits, + entries_length: 0, + entries_byte_length: 0, }; - (header, header_bits) + OplogOpenOutcome::new(oplog, header, Box::new([])) + } else if let Some(mut h2_outcome) = h2_outcome { + // This shouldn't happen because the first header is saved to the first slot + // but Javascript supports this so we should too. + let header_bits: [bool; 2] = [!h2_outcome.header_bit, h2_outcome.header_bit]; + let oplog = Oplog { + header_bits, + entries_length: 0, + entries_byte_length: 0, + }; + OplogOpenOutcome::new(oplog, h2_outcome.state.decode(&existing), Box::new([])) } else { - ( - h1_outcome.state.decode(&existing), - [h1_outcome.header_bit, h1_outcome.header_bit], - ) + // There is nothing in the oplog, start from new. + Self::new(key_pair.clone()) }; - let oplog = Oplog { - header_bits, - entries_length: 0, - entries_byte_length: 0, - }; - OplogOpenOutcome::new(oplog, header, Box::new([])) - } else if let Some(mut h2_outcome) = h2_outcome { - // This shouldn't happen because the first header is saved to the first slot - // but Javascript supports this so we should too. - let header_bits: [bool; 2] = [!h2_outcome.header_bit, h2_outcome.header_bit]; - let oplog = Oplog { - header_bits, - entries_length: 0, - entries_byte_length: 0, - }; - OplogOpenOutcome::new(oplog, h2_outcome.state.decode(&existing), Box::new([])) - } else { - // There is nothing in the oplog, start from new. - Self::new(key_pair) - }; - // Read headers that might be stored in the existing content - if existing.len() > OplogSlot::Entries as usize { - let mut entry_offset = OplogSlot::Entries as usize; - let mut entries: Vec = Vec::new(); - let mut partials: Vec = Vec::new(); - loop { - if let Some(mut entry_outcome) = - Self::validate_leader(entry_offset as usize, &existing)? - { - let entry: Entry = entry_outcome.state.decode(&existing); - entries.push(entry); - partials.push(entry_outcome.partial_bit); - entry_offset = entry_outcome.state.end; - } else { - break; + // Read headers that might be stored in the existing content + if existing.len() > OplogSlot::Entries as usize { + let mut entry_offset = OplogSlot::Entries as usize; + let mut entries: Vec = Vec::new(); + let mut partials: Vec = Vec::new(); + loop { + if let Some(mut entry_outcome) = + Self::validate_leader(entry_offset as usize, &existing)? + { + let entry: Entry = entry_outcome.state.decode(&existing); + entries.push(entry); + partials.push(entry_outcome.partial_bit); + entry_offset = entry_outcome.state.end; + } else { + break; + } + } + + // Remove all trailing partial entries + while partials.len() > 0 && partials[partials.len() - 1] { + entries.pop(); + } + outcome.entries = Some(entries.into_boxed_slice()); } + Ok(Either::Right(outcome)) } - - // Remove all trailing partial entries - while partials.len() > 0 && partials[partials.len() - 1] { - entries.pop(); - } - outcome.entries = Some(entries.into_boxed_slice()); } - Ok(outcome) } /// Appends a changeset to the Oplog. From 3f65c759f1717c2ddc08411abf6dd55fd7c0c751 Mon Sep 17 00:00:00 2001 From: Timo Tiuraniemi Date: Wed, 14 Sep 2022 10:50:39 +0300 Subject: [PATCH 058/157] Add delete test --- tests/js/interop.js | 21 +++++++++++++++++++++ tests/js_interop.rs | 19 +++++++++++++++++++ 2 files changed, 40 insertions(+) diff --git a/tests/js/interop.js b/tests/js/interop.js index 4b97a8bd..82923bdd 100644 --- a/tests/js/interop.js +++ b/tests/js/interop.js @@ -38,6 +38,10 @@ if (process.argv[2] === '1') { step4AppendWithFlush(process.argv[3]).then(result => { console.log("step4 ready", result); }); +} else if (process.argv[2] === '5'){ + step5ClearSome(process.argv[3]).then(result => { + console.log("step5 ready", result); + }); } else { console.error(`Invalid test step {}`, process.argv[2]); process.exit(2); @@ -94,6 +98,23 @@ async function step4AppendWithFlush(testSet) { } } +async function step5ClearSome(testSet) { + const core = new Hypercore(`work/${testSet}`, testKeyPair.publicKey, {keyPair: testKeyPair}); + await core.clear(2); + await core.clear(6, 8); + let info = await core.info(); + assert(info.length, 11); + assert(info.byteLength, 37); + let missing = await core.get(2, { wait: false }); + assert(missing, null); + missing = await core.get(6, { wait: false }); + assert(missing, null); + missing = await core.get(7, { wait: false }); + assert(missing, null); + const second = (await core.get(3)).toString(); + assert(second, "second"); +} + function assert(real, expected) { if (real != expected) { throw new Error(`Got ${real} but expected ${expected}`); diff --git a/tests/js_interop.rs b/tests/js_interop.rs index 8e99b90a..e6b27332 100644 --- a/tests/js_interop.rs +++ b/tests/js_interop.rs @@ -36,6 +36,8 @@ async fn js_interop_js_first() -> Result<()> { assert_eq!(create_hypercore_hash(&work_dir), step_3_hash()); step_4_append_with_flush(&work_dir).await?; assert_eq!(create_hypercore_hash(&work_dir), step_4_hash()); + js_run_step(5, TEST_SET_JS_FIRST); + assert_eq!(create_hypercore_hash(&work_dir), step_5_hash()); Ok(()) } @@ -54,6 +56,8 @@ async fn js_interop_rs_first() -> Result<()> { assert_eq!(create_hypercore_hash(&work_dir), step_3_hash()); js_run_step(4, TEST_SET_RS_FIRST); assert_eq!(create_hypercore_hash(&work_dir), step_4_hash()); + step_5_clear_some(&work_dir).await?; + assert_eq!(create_hypercore_hash(&work_dir), step_5_hash()); Ok(()) } @@ -113,6 +117,12 @@ async fn step_4_append_with_flush(work_dir: &str) -> Result<()> { Ok(()) } +#[cfg(feature = "v10")] +async fn step_5_clear_some(work_dir: &str) -> Result<()> { + let mut hypercore = get_hypercore(work_dir).await?; + Ok(()) +} + #[cfg(feature = "v10")] async fn get_hypercore(work_dir: &str) -> Result> { let path = Path::new(work_dir).to_owned(); @@ -165,3 +175,12 @@ fn step_4_hash() -> common::HypercoreHash { tree: Some("4F346485415AE9A068490764F85CA6307E351C0C8DBD4192F16A9608F5D6F339".into()), } } + +fn step_5_hash() -> common::HypercoreHash { + common::HypercoreHash { + bitfield: Some("F6B695457643DBC7A72787456A75044818A76C95F091067B155B18E9A8ADEAD7".into()), + data: Some("DCAD0C96096AE5F966886FD695CD767832E3ABCB5971782927860799C24866F1".into()), + oplog: Some("F32F8C0564C3F8449952D220CD6AC9C05A071CCF12270059E8F452E3AB0C6F6A".into()), + tree: Some("4F346485415AE9A068490764F85CA6307E351C0C8DBD4192F16A9608F5D6F339".into()), + } +} From d7bcaf11d90312ed16ec0a94c2c13c4436466628 Mon Sep 17 00:00:00 2001 From: Timo Tiuraniemi Date: Wed, 14 Sep 2022 11:32:04 +0300 Subject: [PATCH 059/157] Add one big file to test sparse deletion --- tests/js/interop.js | 31 +++++++++++++++++-------------- tests/js_interop.rs | 31 ++++++++++++++++--------------- 2 files changed, 33 insertions(+), 29 deletions(-) diff --git a/tests/js/interop.js b/tests/js/interop.js index 82923bdd..82de6fe7 100644 --- a/tests/js/interop.js +++ b/tests/js/interop.js @@ -72,20 +72,23 @@ async function step3ReadAndAppendUnflushed(testSet) { result = await core.append([Buffer.from('second'), Buffer.from('third')]); assert(result.length, 5); assert(result.byteLength, 26); - result = await core.append(Buffer.from('fourth')); + const multiBlock = Buffer.alloc(512*3, 'a'); + result = await core.append(multiBlock); assert(result.length, 6); - assert(result.byteLength, 32); + assert(result.byteLength, 1562); result = await core.append([]); assert(result.length, 6); - assert(result.byteLength, 32); + assert(result.byteLength, 1562); const first = (await core.get(2)).toString(); assert(first, "first"); const second = (await core.get(3)).toString(); assert(second, "second"); const third = (await core.get(4)).toString(); assert(third, "third"); - const fourth = (await core.get(5)).toString(); - assert(fourth, "fourth"); + const multiBlockRead = await core.get(5); + if (!multiBlockRead.equals(multiBlock)) { + throw new Error(`Read buffers don't equal, ${multiBlockRead} but expected ${multiBlock}`); + } await core.close(); }; @@ -94,25 +97,25 @@ async function step4AppendWithFlush(testSet) { for (let i=0; i<5; i++) { result = await core.append(Buffer.from([i])); assert(result.length, 6+i+1); - assert(result.byteLength, 32+i+1); + assert(result.byteLength, 1562+i+1); } } async function step5ClearSome(testSet) { const core = new Hypercore(`work/${testSet}`, testKeyPair.publicKey, {keyPair: testKeyPair}); - await core.clear(2); - await core.clear(6, 8); + await core.clear(5); + await core.clear(7, 9); let info = await core.info(); assert(info.length, 11); - assert(info.byteLength, 37); - let missing = await core.get(2, { wait: false }); - assert(missing, null); - missing = await core.get(6, { wait: false }); + assert(info.byteLength, 1567); + let missing = await core.get(5, { wait: false }); assert(missing, null); missing = await core.get(7, { wait: false }); assert(missing, null); - const second = (await core.get(3)).toString(); - assert(second, "second"); + missing = await core.get(8, { wait: false }); + assert(missing, null); + const third = (await core.get(4)).toString(); + assert(third, "third"); } function assert(real, expected) { diff --git a/tests/js_interop.rs b/tests/js_interop.rs index e6b27332..9dff4c05 100644 --- a/tests/js_interop.rs +++ b/tests/js_interop.rs @@ -89,20 +89,21 @@ async fn step_3_read_and_append_unflushed(work_dir: &str) -> Result<()> { let append_outcome = hypercore.append_batch(&[b"second", b"third"]).await?; assert_eq!(append_outcome.length, 5); assert_eq!(append_outcome.byte_length, 26); - let append_outcome = hypercore.append(b"fourth").await?; + let multi_block = &[0x61 as u8; 512 * 3]; + let append_outcome = hypercore.append(multi_block).await?; assert_eq!(append_outcome.length, 6); - assert_eq!(append_outcome.byte_length, 32); + assert_eq!(append_outcome.byte_length, 1562); let append_outcome = hypercore.append_batch(&[]).await?; assert_eq!(append_outcome.length, 6); - assert_eq!(append_outcome.byte_length, 32); + assert_eq!(append_outcome.byte_length, 1562); let first = hypercore.get(2).await?; assert_eq!(first.unwrap(), b"first"); let second = hypercore.get(3).await?; assert_eq!(second.unwrap(), b"second"); let third = hypercore.get(4).await?; assert_eq!(third.unwrap(), b"third"); - let fourth = hypercore.get(5).await?; - assert_eq!(fourth.unwrap(), b"fourth"); + let multi_block_read = hypercore.get(5).await?; + assert_eq!(multi_block_read.unwrap(), multi_block); Ok(()) } @@ -112,7 +113,7 @@ async fn step_4_append_with_flush(work_dir: &str) -> Result<()> { for i in 0..5 { let append_outcome = hypercore.append(&[i]).await?; assert_eq!(append_outcome.length, (6 + i + 1) as u64); - assert_eq!(append_outcome.byte_length, (32 + i + 1) as u64); + assert_eq!(append_outcome.byte_length, (1562 + i as u64 + 1) as u64); } Ok(()) } @@ -161,8 +162,8 @@ fn step_2_hash() -> common::HypercoreHash { fn step_3_hash() -> common::HypercoreHash { common::HypercoreHash { bitfield: Some("DEC1593A7456C8C9407B9B8B9C89682DFFF33C3892BCC9D9F06956FEE0A1B949".into()), - data: Some("A9C34DA27BF72075C2435F8D4EE2DEC75F7AD1ADB31CE4782AFBBC0C6FDEDF1F".into()), - oplog: Some("94E4E7CFB873212B7A38EFAEC4B0BB5426741793ADB9EFC15484C0CEBBD6012B".into()), + data: Some("174BAA59744B5907E90141136A004AE181BE2AC5C2BE569D2FB5D6F182138899".into()), + oplog: Some("86A6CE89669506101DA92D8D18881655CE0606AF8ECEE9CBA2696FE8F328ABD6".into()), tree: Some("38788609A8634DC8D34F9AE723F3169ADB20768ACFDFF266A43B7E217750DD1E".into()), } } @@ -170,17 +171,17 @@ fn step_3_hash() -> common::HypercoreHash { fn step_4_hash() -> common::HypercoreHash { common::HypercoreHash { bitfield: Some("9B844E9378A7D13D6CDD4C1FF12FB313013E5CC472C6CB46497033563FE6B8F1".into()), - data: Some("ADB6D70826037B3E24EB7A9D8BEE314B6B4596812E5FE9C737EB883CB584EDC2".into()), - oplog: Some("39B68580C64A0F96599011E25C539B9F7F89276586DCF12A1F0B1C6446F0D024".into()), - tree: Some("4F346485415AE9A068490764F85CA6307E351C0C8DBD4192F16A9608F5D6F339".into()), + data: Some("963FF6E1FE24CB9E645067A8B70C6B87B9E1F1D244FCFCB15D3B8FA0E18E112E".into()), + oplog: Some("8415E93F229231CEC9DE9FBEAC562F30CC822BCF95C53C992AF94919B898A70F".into()), + tree: Some("263973332D360DC17D025612DCBDC53CBB50BC7554814178DA1783B9CFBB4EEB".into()), } } fn step_5_hash() -> common::HypercoreHash { common::HypercoreHash { - bitfield: Some("F6B695457643DBC7A72787456A75044818A76C95F091067B155B18E9A8ADEAD7".into()), - data: Some("DCAD0C96096AE5F966886FD695CD767832E3ABCB5971782927860799C24866F1".into()), - oplog: Some("F32F8C0564C3F8449952D220CD6AC9C05A071CCF12270059E8F452E3AB0C6F6A".into()), - tree: Some("4F346485415AE9A068490764F85CA6307E351C0C8DBD4192F16A9608F5D6F339".into()), + bitfield: Some("40C9CED82AE0B7A397C9FDD14EEB7F70B74E8F1229F3ED931852591972DDC3E0".into()), + data: Some("0A7715E260F09879C066053497938FF4BA0EA2FCD5DB3F6723BCF69760F2B639".into()), + oplog: Some("2C48E0C47A285C5D764EF7C279FFF23748BCB8B28235B606BE91FF4352F78923".into()), + tree: Some("263973332D360DC17D025612DCBDC53CBB50BC7554814178DA1783B9CFBB4EEB".into()), } } From 4b48764bf0029986543bb06cc2173c511e3b889d Mon Sep 17 00:00:00 2001 From: Timo Tiuraniemi Date: Wed, 14 Sep 2022 11:47:16 +0300 Subject: [PATCH 060/157] Block size seems to be 4096 --- tests/js/interop.js | 10 +++++----- tests/js_interop.rs | 26 +++++++++++++------------- 2 files changed, 18 insertions(+), 18 deletions(-) diff --git a/tests/js/interop.js b/tests/js/interop.js index 82de6fe7..ea0127c8 100644 --- a/tests/js/interop.js +++ b/tests/js/interop.js @@ -72,13 +72,13 @@ async function step3ReadAndAppendUnflushed(testSet) { result = await core.append([Buffer.from('second'), Buffer.from('third')]); assert(result.length, 5); assert(result.byteLength, 26); - const multiBlock = Buffer.alloc(512*3, 'a'); + const multiBlock = Buffer.alloc(4096*3, 'a'); result = await core.append(multiBlock); assert(result.length, 6); - assert(result.byteLength, 1562); + assert(result.byteLength, 12314); result = await core.append([]); assert(result.length, 6); - assert(result.byteLength, 1562); + assert(result.byteLength, 12314); const first = (await core.get(2)).toString(); assert(first, "first"); const second = (await core.get(3)).toString(); @@ -97,7 +97,7 @@ async function step4AppendWithFlush(testSet) { for (let i=0; i<5; i++) { result = await core.append(Buffer.from([i])); assert(result.length, 6+i+1); - assert(result.byteLength, 1562+i+1); + assert(result.byteLength, 12314+i+1); } } @@ -107,7 +107,7 @@ async function step5ClearSome(testSet) { await core.clear(7, 9); let info = await core.info(); assert(info.length, 11); - assert(info.byteLength, 1567); + assert(info.byteLength, 12319); let missing = await core.get(5, { wait: false }); assert(missing, null); missing = await core.get(7, { wait: false }); diff --git a/tests/js_interop.rs b/tests/js_interop.rs index 9dff4c05..68a3ae80 100644 --- a/tests/js_interop.rs +++ b/tests/js_interop.rs @@ -57,7 +57,7 @@ async fn js_interop_rs_first() -> Result<()> { js_run_step(4, TEST_SET_RS_FIRST); assert_eq!(create_hypercore_hash(&work_dir), step_4_hash()); step_5_clear_some(&work_dir).await?; - assert_eq!(create_hypercore_hash(&work_dir), step_5_hash()); + // assert_eq!(create_hypercore_hash(&work_dir), step_5_hash()); Ok(()) } @@ -89,13 +89,13 @@ async fn step_3_read_and_append_unflushed(work_dir: &str) -> Result<()> { let append_outcome = hypercore.append_batch(&[b"second", b"third"]).await?; assert_eq!(append_outcome.length, 5); assert_eq!(append_outcome.byte_length, 26); - let multi_block = &[0x61 as u8; 512 * 3]; + let multi_block = &[0x61 as u8; 4096 * 3]; let append_outcome = hypercore.append(multi_block).await?; assert_eq!(append_outcome.length, 6); - assert_eq!(append_outcome.byte_length, 1562); + assert_eq!(append_outcome.byte_length, 12314); let append_outcome = hypercore.append_batch(&[]).await?; assert_eq!(append_outcome.length, 6); - assert_eq!(append_outcome.byte_length, 1562); + assert_eq!(append_outcome.byte_length, 12314); let first = hypercore.get(2).await?; assert_eq!(first.unwrap(), b"first"); let second = hypercore.get(3).await?; @@ -113,7 +113,7 @@ async fn step_4_append_with_flush(work_dir: &str) -> Result<()> { for i in 0..5 { let append_outcome = hypercore.append(&[i]).await?; assert_eq!(append_outcome.length, (6 + i + 1) as u64); - assert_eq!(append_outcome.byte_length, (1562 + i as u64 + 1) as u64); + assert_eq!(append_outcome.byte_length, (12314 + i as u64 + 1) as u64); } Ok(()) } @@ -162,8 +162,8 @@ fn step_2_hash() -> common::HypercoreHash { fn step_3_hash() -> common::HypercoreHash { common::HypercoreHash { bitfield: Some("DEC1593A7456C8C9407B9B8B9C89682DFFF33C3892BCC9D9F06956FEE0A1B949".into()), - data: Some("174BAA59744B5907E90141136A004AE181BE2AC5C2BE569D2FB5D6F182138899".into()), - oplog: Some("86A6CE89669506101DA92D8D18881655CE0606AF8ECEE9CBA2696FE8F328ABD6".into()), + data: Some("99EB5BC150A1102A7E50D15F90594660010B7FE719D54129065D1D417AA5015A".into()), + oplog: Some("6CC9AB30A4146955886937FB442B86716C1E3C3517772EADA62ACF22D4922EAE".into()), tree: Some("38788609A8634DC8D34F9AE723F3169ADB20768ACFDFF266A43B7E217750DD1E".into()), } } @@ -171,17 +171,17 @@ fn step_3_hash() -> common::HypercoreHash { fn step_4_hash() -> common::HypercoreHash { common::HypercoreHash { bitfield: Some("9B844E9378A7D13D6CDD4C1FF12FB313013E5CC472C6CB46497033563FE6B8F1".into()), - data: Some("963FF6E1FE24CB9E645067A8B70C6B87B9E1F1D244FCFCB15D3B8FA0E18E112E".into()), - oplog: Some("8415E93F229231CEC9DE9FBEAC562F30CC822BCF95C53C992AF94919B898A70F".into()), - tree: Some("263973332D360DC17D025612DCBDC53CBB50BC7554814178DA1783B9CFBB4EEB".into()), + data: Some("AF3AC31CFBE1733C62496CF8E856D5F1EFB4B06CBF1E74204221C89E2F3E1CDE".into()), + oplog: Some("DD00EFAE87A5272281EB040ACCF387D0401895B32DEBE4E9313FCFD0CA2B76AE".into()), + tree: Some("26339A21D606A1F731B90E8001030651D48378116B06A9C1EF87E2538194C2C6".into()), } } fn step_5_hash() -> common::HypercoreHash { common::HypercoreHash { bitfield: Some("40C9CED82AE0B7A397C9FDD14EEB7F70B74E8F1229F3ED931852591972DDC3E0".into()), - data: Some("0A7715E260F09879C066053497938FF4BA0EA2FCD5DB3F6723BCF69760F2B639".into()), - oplog: Some("2C48E0C47A285C5D764EF7C279FFF23748BCB8B28235B606BE91FF4352F78923".into()), - tree: Some("263973332D360DC17D025612DCBDC53CBB50BC7554814178DA1783B9CFBB4EEB".into()), + data: Some("D9FFCCEEE9109751F034ECDAE328672956B90A6E0B409C3173741B8A5D0E75AB".into()), + oplog: Some("AC0E5339F3DC58D7875A60B19D4B3AC7BA34990356CF7B8C1A0BC66FF8F31EFB".into()), + tree: Some("26339A21D606A1F731B90E8001030651D48378116B06A9C1EF87E2538194C2C6".into()), } } From f50c6ac7b37c15f6ea9a75a4d05fc8a37c99caea Mon Sep 17 00:00:00 2001 From: Timo Tiuraniemi Date: Thu, 15 Sep 2022 12:43:34 +0300 Subject: [PATCH 061/157] Use random-access-disk with delete and sparse file support --- Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index 14a8b4ce..5b791861 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -45,7 +45,7 @@ crc32fast = "1.3.2" intmap = { version = "2.0.0", optional = true} [target.'cfg(not(target_arch = "wasm32"))'.dependencies] -random-access-disk = "2.0.0" +random-access-disk = { git = "https://github.com/ttiurani/random-access-disk", branch = "v10" } [dev-dependencies] quickcheck = "0.9.2" From 1efedb4440f01a006f42c469be203cb24646db6b Mon Sep 17 00:00:00 2001 From: Timo Tiuraniemi Date: Fri, 16 Sep 2022 08:59:06 +0300 Subject: [PATCH 062/157] Bitfield index_of and last_index_of --- src/bitfield_v10/dynamic.rs | 167 +++++++++++++++++++++++++++++++++++- src/bitfield_v10/fixed.rs | 41 +++++++++ 2 files changed, 206 insertions(+), 2 deletions(-) diff --git a/src/bitfield_v10/dynamic.rs b/src/bitfield_v10/dynamic.rs index 1405fbd2..c3fbdc6b 100644 --- a/src/bitfield_v10/dynamic.rs +++ b/src/bitfield_v10/dynamic.rs @@ -1,4 +1,4 @@ -use super::fixed::{FixedBitfield, FIXED_BITFIELD_LENGTH}; +use super::fixed::{FixedBitfield, FIXED_BITFIELD_BITS_LENGTH, FIXED_BITFIELD_LENGTH}; use crate::{ common::{StoreInfo, StoreInfoInstruction, StoreInfoType}, Store, @@ -15,6 +15,7 @@ const DYNAMIC_BITFIELD_PAGE_SIZE: usize = 32768; #[derive(Debug)] pub struct DynamicBitfield { pages: intmap::IntMap>, + biggest_page_index: u64, unflushed: Vec, } @@ -35,6 +36,7 @@ impl DynamicBitfield { } let data = info.data.expect("Did not receive bitfield store content"); let resumed = data.len() >= 4; + let mut biggest_page_index = 0; if resumed { let mut pages: intmap::IntMap> = intmap::IntMap::new(); let mut data_index = 0; @@ -44,16 +46,21 @@ impl DynamicBitfield { parent_index, RefCell::new(FixedBitfield::from_data(parent_index, data_index, &data)), ); + if parent_index > biggest_page_index { + biggest_page_index = parent_index; + } data_index += FIXED_BITFIELD_LENGTH; } Either::Right(Self { pages, unflushed: vec![], + biggest_page_index, }) } else { Either::Right(Self { pages: intmap::IntMap::new(), unflushed: vec![], + biggest_page_index, }) } } @@ -94,7 +101,15 @@ impl DynamicBitfield { let i = (index - j) / DYNAMIC_BITFIELD_PAGE_SIZE as u64; if !self.pages.contains_key(i) { - self.pages.insert(i, RefCell::new(FixedBitfield::new(i))); + if value { + self.pages.insert(i, RefCell::new(FixedBitfield::new(i))); + if i > self.biggest_page_index { + self.biggest_page_index = i; + } + } else { + // The page does not exist, but when setting false, that doesn't matter + return false; + } } let mut p = self.pages.get_mut(i).unwrap().borrow_mut(); @@ -115,6 +130,9 @@ impl DynamicBitfield { while length > 0 { if !self.pages.contains_key(i) { self.pages.insert(i, RefCell::new(FixedBitfield::new(i))); + if i > self.biggest_page_index { + self.biggest_page_index = i; + } } let mut p = self.pages.get_mut(i).unwrap().borrow_mut(); @@ -138,6 +156,116 @@ impl DynamicBitfield { length -= range_end as u64; } } + + /// Finds the first index of the value after given position. Returns None if not found. + pub fn index_of(&self, value: bool, position: u64) -> Option { + let first_index = position & (DYNAMIC_BITFIELD_PAGE_SIZE as u64 - 1); + let first_page = (position - first_index) / (DYNAMIC_BITFIELD_PAGE_SIZE as u64); + + if value { + // For finding the first positive value, we only care about pages that are set, + // not pages that don't exist, as they can't possibly contain the value. + + // To keep the common case fast, first try the same page as the position + if let Some(p) = self.pages.get(first_page) { + if let Some(index) = p.borrow().index_of(value, first_index as u32) { + return Some(first_page * DYNAMIC_BITFIELD_PAGE_SIZE as u64 + index as u64); + }; + } + + // It wasn't found on the first page, now get the keys that are bigger + // than the given index and sort them. + let mut keys: Vec<&u64> = self + .pages + .keys() + .into_iter() + .filter(|key| **key > first_page) + .collect(); + keys.sort(); + for key in keys { + if let Some(p) = self.pages.get(*key) { + if let Some(index) = p.borrow().index_of(value, 0) { + return Some(key * DYNAMIC_BITFIELD_PAGE_SIZE as u64 + index as u64); + }; + } + } + } else { + // Searching for the false value is easier as it is automatically hit on + // a missing page. + let mut i = first_page; + let mut j = first_index as u32; + while i == first_page || i <= self.biggest_page_index { + if let Some(p) = self.pages.get(i) { + if let Some(index) = p.borrow().index_of(value, j) { + return Some(i * DYNAMIC_BITFIELD_PAGE_SIZE as u64 + index as u64); + }; + } else { + return Some(i * DYNAMIC_BITFIELD_PAGE_SIZE as u64 + j as u64); + } + i += 1; + j = 0; // We start at the beginning of each page + } + } + None + } + + /// Finds the last index of the value before given position. Returns None if not found. + pub fn last_index_of(&self, value: bool, position: u64) -> Option { + let last_index = position & (DYNAMIC_BITFIELD_PAGE_SIZE as u64 - 1); + let last_page = (position - last_index) / (DYNAMIC_BITFIELD_PAGE_SIZE as u64); + + if value { + // For finding the last positive value, we only care about pages that are set, + // not pages that don't exist, as they can't possibly contain the value. + + // To keep the common case fast, first try the same page as the position + if let Some(p) = self.pages.get(last_page) { + if let Some(index) = p.borrow().last_index_of(value, last_index as u32) { + return Some(last_page * DYNAMIC_BITFIELD_PAGE_SIZE as u64 + index as u64); + }; + } + + // It wasn't found on the last page, now get the keys that are smaller + // than the given index and sort them. + let mut keys: Vec<&u64> = self + .pages + .keys() + .into_iter() + .filter(|key| **key < last_page) + .collect(); + keys.sort(); + keys.reverse(); + + for key in keys { + if let Some(p) = self.pages.get(*key) { + if let Some(index) = p + .borrow() + .last_index_of(value, FIXED_BITFIELD_BITS_LENGTH as u32 - 1) + { + return Some(key * DYNAMIC_BITFIELD_PAGE_SIZE as u64 + index as u64); + }; + } + } + } else { + // Searching for the false value is easier as it is automatically hit on + // a missing page. + let mut i = last_page; + let mut j = last_index as u32; + while i == last_page || i <= 0 { + if let Some(p) = self.pages.get(i) { + if let Some(index) = p.borrow().last_index_of(value, j) { + return Some(i * DYNAMIC_BITFIELD_PAGE_SIZE as u64 + index as u64); + }; + } else { + return Some(i * DYNAMIC_BITFIELD_PAGE_SIZE as u64 + j as u64); + } + i -= 1; + j = FIXED_BITFIELD_BITS_LENGTH as u32 - 1; // We start at end of each page + } + } + + None + } } #[cfg(test)] @@ -161,8 +289,23 @@ mod tests { fn bitfield_dynamic_get_and_set() { let mut bitfield = get_dynamic_bitfield(); assert_value_range(&bitfield, 0, 9, false); + assert_eq!(bitfield.index_of(true, 0), None); + assert_eq!(bitfield.index_of(false, 0), Some(0)); + assert_eq!(bitfield.last_index_of(true, 9), None); + assert_eq!(bitfield.last_index_of(false, 9), Some(9)); + assert_eq!(bitfield.index_of(true, 10000000), None); + assert_eq!(bitfield.index_of(false, 10000000), Some(10000000)); + assert_eq!(bitfield.last_index_of(true, 10000000), None); + assert_eq!(bitfield.last_index_of(false, 10000000), Some(10000000)); + bitfield.set(0, true); assert_eq!(bitfield.get(0), true); + assert_eq!(bitfield.index_of(true, 0), Some(0)); + assert_eq!(bitfield.index_of(false, 0), Some(1)); + assert_eq!(bitfield.last_index_of(true, 9), Some(0)); + assert_eq!(bitfield.last_index_of(false, 9), Some(9)); + assert_eq!(bitfield.last_index_of(true, 10000000), Some(0)); + assert_eq!(bitfield.last_index_of(false, 10000000), Some(10000000)); assert_value_range(&bitfield, 1, 63, false); bitfield.set(31, true); @@ -189,6 +332,9 @@ mod tests { assert_eq!(bitfield.get(10000000), true); assert_value_range(&bitfield, 9999990, 10, false); assert_value_range(&bitfield, 10000001, 9, false); + assert_eq!(bitfield.index_of(false, 32767), Some(32769)); + assert_eq!(bitfield.index_of(true, 32769), Some(10000000)); + assert_eq!(bitfield.last_index_of(true, 9999999), Some(32768)); } #[test] @@ -231,6 +377,23 @@ mod tests { bitfield.set_range(10000000, 50, true); assert_value_range(&bitfield, 9999990, 9, false); assert_value_range(&bitfield, 10000050, 9, false); + assert_eq!(bitfield.index_of(true, 32780), Some(10000000)); + bitfield.set_range(0, 32780, false); + // Manufacture empty pages to test sorting + bitfield.set(900000, true); + bitfield.set(900000, false); + bitfield.set(300000, true); + bitfield.set(300000, false); + bitfield.set(200000, true); + bitfield.set(200000, false); + bitfield.set(500000, true); + bitfield.set(500000, false); + bitfield.set(100000, true); + bitfield.set(100000, false); + bitfield.set(700000, true); + bitfield.set(700000, false); + assert_eq!(bitfield.index_of(true, 0), Some(10000000)); + assert_eq!(bitfield.last_index_of(true, 9999999), None); bitfield.set_range(10000010, 10, false); assert_value_range(&bitfield, 10000000, 10, true); diff --git a/src/bitfield_v10/fixed.rs b/src/bitfield_v10/fixed.rs index 9b8113d0..fc827058 100644 --- a/src/bitfield_v10/fixed.rs +++ b/src/bitfield_v10/fixed.rs @@ -1,7 +1,9 @@ pub(crate) const FIXED_BITFIELD_LENGTH: usize = 1024; pub(crate) const FIXED_BITFIELD_BYTES_LENGTH: usize = FIXED_BITFIELD_LENGTH * 4; +pub(crate) const FIXED_BITFIELD_BITS_LENGTH: usize = FIXED_BITFIELD_BYTES_LENGTH * 8; // u32 has 4 bytes and a byte has 8 bits const FIXED_BITFIELD_BITS_PER_ELEM: u32 = 4 * 8; + use std::convert::TryInto; /// Fixed size bitfield @@ -133,6 +135,26 @@ impl FixedBitfield { changed } + + /// Finds the first index of the value after given position. Returns None if not found. + pub fn index_of(&self, value: bool, position: u32) -> Option { + for i in position..FIXED_BITFIELD_BITS_LENGTH as u32 { + if self.get(i) == value { + return Some(i); + } + } + None + } + + /// Finds the last index of the value before given position. Returns None if not found. + pub fn last_index_of(&self, value: bool, position: u32) -> Option { + for i in (0..position + 1).rev() { + if self.get(i) == value { + return Some(i); + } + } + None + } } #[cfg(test)] @@ -149,12 +171,24 @@ mod tests { fn bitfield_fixed_get_and_set() { let mut bitfield = FixedBitfield::new(0); assert_value_range(&bitfield, 0, 9, false); + assert_eq!(bitfield.index_of(true, 0), None); + assert_eq!(bitfield.index_of(false, 0), Some(0)); + assert_eq!(bitfield.last_index_of(true, 9), None); + assert_eq!(bitfield.last_index_of(false, 9), Some(9)); + bitfield.set(0, true); assert_eq!(bitfield.get(0), true); + assert_eq!(bitfield.index_of(true, 0), Some(0)); + assert_eq!(bitfield.index_of(false, 0), Some(1)); + assert_eq!(bitfield.last_index_of(true, 9), Some(0)); + assert_eq!(bitfield.last_index_of(false, 9), Some(9)); + assert_eq!(bitfield.last_index_of(false, 0), None); assert_value_range(&bitfield, 1, 63, false); bitfield.set(31, true); assert_eq!(bitfield.get(31), true); + assert_eq!(bitfield.index_of(true, 1), Some(31)); + assert_eq!(bitfield.index_of(false, 31), Some(32)); assert_value_range(&bitfield, 32, 32, false); assert_eq!(bitfield.get(32), false); @@ -167,6 +201,9 @@ mod tests { bitfield.set(32767, true); assert_eq!(bitfield.get(32767), true); assert_value_range(&bitfield, 32760, 7, false); + assert_eq!(bitfield.index_of(true, 33), Some(32767)); + assert_eq!(bitfield.last_index_of(true, 9), Some(0)); + assert_eq!(bitfield.last_index_of(true, 32766), Some(32)); } #[test] @@ -191,6 +228,10 @@ mod tests { assert_value_range(&bitfield, 30, 100, true); assert_value_range(&bitfield, 30050, 50, true); assert_value_range(&bitfield, 31000, 50, false); + assert_eq!(bitfield.index_of(true, 20), Some(30)); + assert_eq!(bitfield.index_of(false, 30), Some(30100)); + assert_eq!(bitfield.last_index_of(true, 32000), Some(30099)); + assert_eq!(bitfield.last_index_of(false, 30099), Some(29)); bitfield.set_range(32750, 18, true); assert_value_range(&bitfield, 32750, 18, true); From 201e7baaafc4495ced52e7810bcad169330464ac Mon Sep 17 00:00:00 2001 From: Timo Tiuraniemi Date: Fri, 16 Sep 2022 09:44:26 +0300 Subject: [PATCH 063/157] Clearing works, but random-access-disk still holepunches differently than JS --- src/common/store.rs | 11 +++++ src/core.rs | 92 +++++++++++++++++++++++++++++++++++++++++ src/data/mod.rs | 5 +++ src/oplog/mod.rs | 17 +++++++- src/storage_v10/mod.rs | 11 ++++- src/tree/merkle_tree.rs | 72 ++++++++++++++++++++------------ tests/js/interop.js | 3 ++ tests/js_interop.rs | 16 ++++++- 8 files changed, 196 insertions(+), 31 deletions(-) diff --git a/src/common/store.rs b/src/common/store.rs index ef9c900b..34001fd2 100644 --- a/src/common/store.rs +++ b/src/common/store.rs @@ -44,6 +44,17 @@ impl StoreInfo { } } + pub fn new_delete(store: Store, index: u64, length: u64) -> Self { + Self { + store, + info_type: StoreInfoType::Content, + index, + length: Some(length), + data: None, + drop: true, + } + } + pub fn new_truncate(store: Store, index: u64) -> Self { Self { store, diff --git a/src/core.rs b/src/core.rs index 912dec48..b35edd2b 100644 --- a/src/core.rs +++ b/src/core.rs @@ -2,6 +2,7 @@ use crate::{ bitfield_v10::Bitfield, + common::StoreInfo, crypto::generate_keypair, data::BlockStore, oplog::{Header, Oplog, MAX_OPLOG_ENTRIES_BYTE_SIZE}, @@ -37,6 +38,14 @@ pub struct AppendOutcome { pub byte_length: u64, } +/// Info about the hypercore +#[derive(Debug)] +pub struct Info { + pub length: u64, + pub byte_length: u64, + pub contiguous_length: u64, +} + impl Hypercore where T: RandomAccess> + Debug + Send, @@ -177,6 +186,15 @@ where }) } + /// Gets basic info about the Hypercore + pub fn info(&self) -> Info { + Info { + length: self.tree.length, + byte_length: self.tree.byte_length, + contiguous_length: self.header.contiguous_length, + } + } + /// Appends a data slice to the hypercore. pub async fn append(&mut self, data: &[u8]) -> Result { self.append_batch(&[data]).await @@ -278,6 +296,80 @@ where Ok(Some(data.to_vec())) } + /// Clear data for entries between start and end (exclusive) indexes. + pub async fn clear(&mut self, start: u64, end: u64) -> Result<()> { + if start >= end { + // NB: This is what javascript does, so we mimic that here + return Ok(()); + } + // Write to oplog + let infos_to_flush = self.oplog.clear(start, end); + self.storage.flush_infos(&infos_to_flush).await?; + + // Set bitfield + self.bitfield.set_range(start, end - start, false); + + // Set contiguous length + if start < self.header.contiguous_length { + self.header.contiguous_length = start; + } + + // Find the biggest hole that can be punched into the data + let start = if let Some(index) = self.bitfield.last_index_of(true, start) { + index + 1 + } else { + 0 + }; + let end = if let Some(index) = self.bitfield.index_of(true, end) { + index + } else { + self.tree.length + }; + + // Find byte offset for first value + let mut infos: Vec = Vec::new(); + let clear_offset = match self.tree.byte_offset(start, None)? { + Either::Right(value) => value, + Either::Left(instructions) => { + let new_infos = self.storage.read_infos_to_vec(&instructions).await?; + infos.extend(new_infos); + match self.tree.byte_offset(start, Some(&infos))? { + Either::Right(value) => value, + Either::Left(_) => { + return Err(anyhow!("Could not read offset for index")); + } + } + } + }; + + // Find byte range for last value + let last_byte_range = match self.tree.byte_range(end - 1, Some(&infos))? { + Either::Right(value) => value, + Either::Left(instructions) => { + let new_infos = self.storage.read_infos_to_vec(&instructions).await?; + infos.extend(new_infos); + match self.tree.byte_range(end - 1, Some(&infos))? { + Either::Right(value) => value, + Either::Left(_) => { + return Err(anyhow!("Could not read byte range")); + } + } + } + }; + let clear_length = (last_byte_range.index + last_byte_range.length) - clear_offset; + + // Clear blocks + let info_to_flush = self.block_store.clear(clear_offset, clear_length); + self.storage.flush_info(info_to_flush).await?; + + // Now ready to flush + if self.should_flush_bitfield_and_tree_and_oplog() { + self.flush_bitfield_and_tree_and_oplog().await?; + } + + Ok(()) + } + fn should_flush_bitfield_and_tree_and_oplog(&mut self) -> bool { if self.skip_flush_count == 0 || self.oplog.entries_byte_length >= MAX_OPLOG_ENTRIES_BYTE_SIZE diff --git a/src/data/mod.rs b/src/data/mod.rs index 5bfc799d..c2c47a25 100644 --- a/src/data/mod.rs +++ b/src/data/mod.rs @@ -34,4 +34,9 @@ impl BlockStore { )) } } + + /// Clears a segment, returns infos to write to storage. + pub fn clear(&mut self, start: u64, length: u64) -> StoreInfo { + StoreInfo::new_delete(Store::Data, start, length) + } } diff --git a/src/oplog/mod.rs b/src/oplog/mod.rs index 03f96876..33243f20 100644 --- a/src/oplog/mod.rs +++ b/src/oplog/mod.rs @@ -210,7 +210,22 @@ impl Oplog { } } - /// Flushes pending changes, returns info slices to write to storage. + /// Clears a segment, returns infos to write to storage. + pub fn clear(&mut self, start: u64, end: u64) -> Box<[StoreInfo]> { + let entry: Entry = Entry { + user_data: vec![], + tree_nodes: vec![], + tree_upgrade: None, + bitfield: Some(EntryBitfieldUpdate { + drop: true, + start, + length: end - start, + }), + }; + self.append_entries(&[entry], false) + } + + /// Flushes pending changes, returns infos to write to storage. pub fn flush(&mut self, header: &Header) -> Box<[StoreInfo]> { let (new_header_bits, infos_to_flush) = Self::insert_header(header, 0, self.header_bits); self.entries_byte_length = 0; diff --git a/src/storage_v10/mod.rs b/src/storage_v10/mod.rs index 4013c696..538e522b 100644 --- a/src/storage_v10/mod.rs +++ b/src/storage_v10/mod.rs @@ -93,7 +93,8 @@ where Ok(infos.into_boxed_slice()) } - async fn read_infos_to_vec( + /// Reads infos but retains them as a Vec + pub async fn read_infos_to_vec( &mut self, info_instructions: &[StoreInfoInstruction], ) -> Result> { @@ -164,7 +165,13 @@ where .map_err(|e| anyhow!(e))?; } } else { - unimplemented!("Deleting not implemented yet") + storage + .del( + info.index, + info.length.expect("When deleting, length must be given"), + ) + .await + .map_err(|e| anyhow!(e))?; } } StoreInfoType::Size => { diff --git a/src/tree/merkle_tree.rs b/src/tree/merkle_tree.rs index 518c6091..7b0a9d35 100644 --- a/src/tree/merkle_tree.rs +++ b/src/tree/merkle_tree.rs @@ -141,23 +141,7 @@ impl MerkleTree { hypercore_index: u64, infos: Option<&[StoreInfo]>, ) -> Result, NodeByteRange>> { - // Converts a hypercore index into a merkle tree index - let index = 2 * hypercore_index; - - // Check bounds - let head = 2 * self.length; - let compare_index = if index & 1 == 0 { - index - } else { - flat_tree::right_span(index) - }; - if compare_index >= head { - return Err(anyhow!( - "Hypercore index {} is out of bounds", - hypercore_index - )); - } - + let index = self.validate_hypercore_index(hypercore_index)?; // Get nodes out of incoming infos let nodes: Vec = infos_to_nodes(infos); @@ -165,9 +149,9 @@ impl MerkleTree { // of the byte range let length_result = self.get_node(index, &nodes)?; - // As for the offset, that might require a lot more nodes to combine into - // an offset - let offset_result = self.byte_offset(index, &nodes)?; + // As for the offset, that might require fetching a lot more nodes whose + // lengths to sum + let offset_result = self.byte_offset_from_nodes(index, &nodes)?; // Construct response of either instructions (Left) or the result (Right) let mut instructions: Vec = Vec::new(); @@ -177,9 +161,6 @@ impl MerkleTree { }; match length_result { Either::Left(instruction) => { - if infos.is_some() { - return Err(anyhow!("Could not return size from fetched nodes")); - } instructions.push(instruction); } Either::Right(node) => { @@ -188,9 +169,6 @@ impl MerkleTree { } match offset_result { Either::Left(offset_instructions) => { - if infos.is_some() { - return Err(anyhow!("Could not return offset from fetched nodes")); - } instructions.extend(offset_instructions); } Either::Right(offset) => { @@ -205,6 +183,25 @@ impl MerkleTree { } } + /// Get the byte offset given hypercore index + pub fn byte_offset( + &self, + hypercore_index: u64, + infos: Option<&[StoreInfo]>, + ) -> Result, u64>> { + let index = self.validate_hypercore_index(hypercore_index)?; + // Get nodes out of incoming infos + let nodes: Vec = infos_to_nodes(infos); + // Get offset + let offset_result = self.byte_offset_from_nodes(index, &nodes)?; + match offset_result { + Either::Left(offset_instructions) => { + Ok(Either::Left(offset_instructions.into_boxed_slice())) + } + Either::Right(offset) => Ok(Either::Right(offset)), + } + } + pub fn add_node(&mut self, node: Node) { self.unflushed.insert(node.index, node); } @@ -347,7 +344,28 @@ impl MerkleTree { infos_to_flush } - fn byte_offset( + /// Validates given hypercore index and returns tree index + fn validate_hypercore_index(&self, hypercore_index: u64) -> Result { + // Converts a hypercore index into a merkle tree index + let index = 2 * hypercore_index; + + // Check bounds + let head = 2 * self.length; + let compare_index = if index & 1 == 0 { + index + } else { + flat_tree::right_span(index) + }; + if compare_index >= head { + return Err(anyhow!( + "Hypercore index {} is out of bounds", + hypercore_index + )); + } + Ok(index) + } + + fn byte_offset_from_nodes( &self, index: u64, nodes: &Vec, diff --git a/tests/js/interop.js b/tests/js/interop.js index ea0127c8..59e9f373 100644 --- a/tests/js/interop.js +++ b/tests/js/interop.js @@ -108,6 +108,9 @@ async function step5ClearSome(testSet) { let info = await core.info(); assert(info.length, 11); assert(info.byteLength, 12319); + assert(info.contiguousLength, 5); + assert(info.padding, 0); + let missing = await core.get(5, { wait: false }); assert(missing, null); missing = await core.get(7, { wait: false }); diff --git a/tests/js_interop.rs b/tests/js_interop.rs index 68a3ae80..86d294d6 100644 --- a/tests/js_interop.rs +++ b/tests/js_interop.rs @@ -57,7 +57,7 @@ async fn js_interop_rs_first() -> Result<()> { js_run_step(4, TEST_SET_RS_FIRST); assert_eq!(create_hypercore_hash(&work_dir), step_4_hash()); step_5_clear_some(&work_dir).await?; - // assert_eq!(create_hypercore_hash(&work_dir), step_5_hash()); + assert_eq!(create_hypercore_hash(&work_dir), step_5_hash()); Ok(()) } @@ -121,6 +121,20 @@ async fn step_4_append_with_flush(work_dir: &str) -> Result<()> { #[cfg(feature = "v10")] async fn step_5_clear_some(work_dir: &str) -> Result<()> { let mut hypercore = get_hypercore(work_dir).await?; + hypercore.clear(5, 6).await?; + hypercore.clear(7, 9).await?; + let info = hypercore.info(); + assert_eq!(info.length, 11); + assert_eq!(info.byte_length, 12319); + assert_eq!(info.contiguous_length, 5); + let missing = hypercore.get(5).await?; + assert_eq!(missing, None); + let missing = hypercore.get(7).await?; + assert_eq!(missing, None); + let missing = hypercore.get(8).await?; + assert_eq!(missing, None); + let third = hypercore.get(4).await?; + assert_eq!(third.unwrap(), b"third"); Ok(()) } From 91acd2cc9a675ad6414db9dd7f8b0bc3bcd5a6d0 Mon Sep 17 00:00:00 2001 From: Timo Tiuraniemi Date: Mon, 19 Sep 2022 11:09:58 +0300 Subject: [PATCH 064/157] Switch to explicit v9 feature to be able to use it from hypercore-protocol --- Cargo.toml | 4 +++- examples/async.rs | 8 ++++---- examples/main.rs | 4 ++-- src/common/node.rs | 12 ++++++------ src/lib.rs | 18 +++++++++--------- src/prelude.rs | 6 +++--- tests/bitfield.rs | 18 +++++++++--------- tests/common/mod.rs | 4 ++-- tests/compat.rs | 14 +++++++------- tests/feed.rs | 36 ++++++++++++++++++------------------ tests/model.rs | 4 ++-- tests/regression.rs | 4 ++-- tests/storage.rs | 12 ++++++------ 13 files changed, 73 insertions(+), 71 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 5b791861..a726ccfd 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -57,7 +57,9 @@ sha2 = "0.10.2" [features] default = ["v10"] -# v10 version of the crate, without this, v9 is built +# v9 version of the crate +v9 = [] +# v10 version of the crate v10 = ["intmap"] # Used only in interoperability tests under tests/js-interop which use the javascript version of hypercore # to verify that this crate works. To run them, use: diff --git a/examples/async.rs b/examples/async.rs index 14564346..2f9ce57a 100644 --- a/examples/async.rs +++ b/examples/async.rs @@ -1,10 +1,10 @@ use async_std::task; -#[cfg(not(feature = "v10"))] +#[cfg(feature = "v9")] use hypercore::Feed; use random_access_storage::RandomAccess; use std::fmt::Debug; -#[cfg(not(feature = "v10"))] +#[cfg(feature = "v9")] async fn append(feed: &mut Feed, content: &[u8]) where T: RandomAccess> + Debug + Send, @@ -12,7 +12,7 @@ where feed.append(content).await.unwrap(); } -#[cfg(not(feature = "v10"))] +#[cfg(feature = "v9")] async fn print(feed: &mut Feed) where T: RandomAccess> + Debug + Send, @@ -21,7 +21,7 @@ where println!("{:?}", feed.get(1).await); } -#[cfg(not(feature = "v10"))] +#[cfg(feature = "v9")] fn main() { task::block_on(task::spawn(async { let mut feed = Feed::default(); diff --git a/examples/main.rs b/examples/main.rs index 8acda765..e5000173 100644 --- a/examples/main.rs +++ b/examples/main.rs @@ -1,8 +1,8 @@ -#[cfg(not(feature = "v10"))] +#[cfg(feature = "v9")] use hypercore::Feed; #[async_std::main] -#[cfg(not(feature = "v10"))] +#[cfg(feature = "v9")] async fn main() { let mut feed = Feed::open("feed.db").await.expect("Failed to create dir"); diff --git a/src/common/node.rs b/src/common/node.rs index 3849b582..61e5c01b 100644 --- a/src/common/node.rs +++ b/src/common/node.rs @@ -1,8 +1,8 @@ -#[cfg(not(feature = "v10"))] // tree module has the byte manipulation for v10 +#[cfg(feature = "v9")] // tree module has the byte manipulation for v10 use anyhow::ensure; -#[cfg(not(feature = "v10"))] // tree module has the byte manipulation for v10 +#[cfg(feature = "v9")] // tree module has the byte manipulation for v10 use anyhow::Result; -#[cfg(not(feature = "v10"))] // tree module has the byte manipulation for v10 +#[cfg(feature = "v9")] // tree module has the byte manipulation for v10 use byteorder::{BigEndian, ReadBytesExt, WriteBytesExt}; use merkle_tree_stream::Node as NodeTrait; use merkle_tree_stream::{NodeKind, NodeParts}; @@ -10,7 +10,7 @@ use pretty_hash::fmt as pretty_fmt; use std::cmp::Ordering; use std::convert::AsRef; use std::fmt::{self, Display}; -#[cfg(not(feature = "v10"))] // tree module has the byte manipulation for v10 +#[cfg(feature = "v9")] // tree module has the byte manipulation for v10 use std::io::Cursor; use crate::crypto::Hash; @@ -65,7 +65,7 @@ impl Node { /// Convert a vector to a new instance. /// /// Requires the index at which the buffer was read to be passed. - #[cfg(not(feature = "v10"))] // tree module has the byte manipulation for v10 + #[cfg(feature = "v9")] // tree module has the byte manipulation for v10 pub fn from_bytes(index: u64, buffer: &[u8]) -> Result { ensure!(buffer.len() == 40, "buffer should be 40 bytes"); @@ -96,7 +96,7 @@ impl Node { } /// Convert to a buffer that can be written to disk. - #[cfg(not(feature = "v10"))] // tree module has the byte manipulation for v10 + #[cfg(feature = "v9")] // tree module has the byte manipulation for v10 pub fn to_bytes(&self) -> Result> { let mut writer = Vec::with_capacity(40); writer.extend_from_slice(&self.hash); diff --git a/src/lib.rs b/src/lib.rs index 33fb281d..9d5bdb91 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -13,7 +13,7 @@ //! //! ## Example //! ```rust -//! #[cfg(not(feature = "v10"))] +//! #[cfg(feature = "v9")] //! # fn main() -> Result<(), Box> { //! # async_std::task::block_on(async { //! let mut feed = hypercore::open("./feed.db").await?; @@ -39,7 +39,7 @@ //! [Dat]: https://github.com/datrs //! [Feed]: crate::feed::Feed -#[cfg(not(feature = "v10"))] +#[cfg(feature = "v9")] pub mod bitfield; pub mod compact_encoding; pub mod prelude; @@ -55,15 +55,15 @@ mod crypto; #[cfg(feature = "v10")] mod data; mod event; -#[cfg(not(feature = "v10"))] +#[cfg(feature = "v9")] mod feed; -#[cfg(not(feature = "v10"))] +#[cfg(feature = "v9")] mod feed_builder; #[cfg(feature = "v10")] mod oplog; mod proof; mod replicate; -#[cfg(not(feature = "v10"))] +#[cfg(feature = "v9")] mod storage; #[cfg(feature = "v10")] mod storage_v10; @@ -77,13 +77,13 @@ pub use crate::common::Store; pub use crate::core::Hypercore; pub use crate::crypto::{generate_keypair, sign, verify, Signature}; pub use crate::event::Event; -#[cfg(not(feature = "v10"))] +#[cfg(feature = "v9")] pub use crate::feed::Feed; -#[cfg(not(feature = "v10"))] +#[cfg(feature = "v9")] pub use crate::feed_builder::FeedBuilder; pub use crate::proof::Proof; pub use crate::replicate::Peer; -#[cfg(not(feature = "v10"))] +#[cfg(feature = "v9")] pub use crate::storage::{NodeTrait, PartialKeypair, Storage, Store}; #[cfg(feature = "v10")] pub use crate::storage_v10::{PartialKeypair, Storage}; @@ -92,7 +92,7 @@ pub use ed25519_dalek::{PublicKey, SecretKey}; use std::path::Path; /// Create a new Hypercore `Feed`. -#[cfg(not(feature = "v10"))] +#[cfg(feature = "v9")] pub async fn open>( path: P, ) -> anyhow::Result> { diff --git a/src/prelude.rs b/src/prelude.rs index 188c3a02..5c2c1980 100644 --- a/src/prelude.rs +++ b/src/prelude.rs @@ -2,10 +2,10 @@ //! //! ```rust //! use hypercore::prelude::*; -//! #[cfg(not(feature = "v10"))] +//! #[cfg(feature = "v9")] //! let feed = Feed::default(); //! ``` -#[cfg(not(feature = "v10"))] +#[cfg(feature = "v9")] pub use crate::feed::Feed; // pub use feed_builder::FeedBuilder; pub use crate::common::Node; @@ -13,7 +13,7 @@ pub use crate::common::Node; pub use crate::common::Store; #[cfg(feature = "v10")] pub use crate::core::Hypercore; -#[cfg(not(feature = "v10"))] +#[cfg(feature = "v9")] pub use crate::storage::{NodeTrait, Storage, Store}; #[cfg(feature = "v10")] pub use crate::storage_v10::{PartialKeypair, Storage}; diff --git a/tests/bitfield.rs b/tests/bitfield.rs index 6225a3d5..4c4ea182 100644 --- a/tests/bitfield.rs +++ b/tests/bitfield.rs @@ -1,12 +1,12 @@ use rand; -#[cfg(not(feature = "v10"))] +#[cfg(feature = "v9")] use hypercore::bitfield::{Bitfield, Change::*}; -#[cfg(not(feature = "v10"))] +#[cfg(feature = "v9")] use rand::Rng; #[test] -#[cfg(not(feature = "v10"))] +#[cfg(feature = "v9")] fn set_and_get() { let (mut b, _) = Bitfield::new(); @@ -22,7 +22,7 @@ fn set_and_get() { } #[test] -#[cfg(not(feature = "v10"))] +#[cfg(feature = "v9")] fn set_and_get_tree() { let (mut b, mut tree) = Bitfield::new(); @@ -43,7 +43,7 @@ fn set_and_get_tree() { } #[test] -#[cfg(not(feature = "v10"))] +#[cfg(feature = "v9")] fn set_and_index() { let (mut b, _) = Bitfield::new(); @@ -102,7 +102,7 @@ fn set_and_index() { } #[test] -#[cfg(not(feature = "v10"))] +#[cfg(feature = "v9")] fn set_and_index_random() { let (mut b, _) = Bitfield::new(); @@ -142,7 +142,7 @@ fn set_and_index_random() { } #[test] -#[cfg(not(feature = "v10"))] +#[cfg(feature = "v9")] fn get_total_positive_bits() { let (mut b, _) = Bitfield::new(); @@ -161,7 +161,7 @@ fn get_total_positive_bits() { } #[test] -#[cfg(not(feature = "v10"))] +#[cfg(feature = "v9")] fn bitfield_dedup() { let (mut b, mut tree) = Bitfield::new(); @@ -181,7 +181,7 @@ fn bitfield_dedup() { } #[test] -#[cfg(not(feature = "v10"))] +#[cfg(feature = "v9")] fn bitfield_compress() { let (mut b, _) = Bitfield::new(); assert_eq!(b.compress(0, 0).unwrap(), vec![0]); diff --git a/tests/common/mod.rs b/tests/common/mod.rs index b82f5caa..a470166c 100644 --- a/tests/common/mod.rs +++ b/tests/common/mod.rs @@ -3,7 +3,7 @@ use hypercore::PartialKeypair; use anyhow::Error; use ed25519_dalek::{PublicKey, SecretKey, PUBLIC_KEY_LENGTH, SECRET_KEY_LENGTH}; use futures::future::FutureExt; -#[cfg(not(feature = "v10"))] +#[cfg(feature = "v9")] use hypercore::{Feed, Storage, Store}; use random_access_memory as ram; use sha2::{Digest, Sha256}; @@ -20,7 +20,7 @@ const TEST_SECRET_KEY_BYTES: [u8; SECRET_KEY_LENGTH] = [ 0x51, 0x0b, 0x71, 0x14, 0x15, 0xf3, 0x31, 0xf6, 0xfa, 0x9e, 0xf2, 0xbf, 0x23, 0x5f, 0x2f, 0xfe, ]; -#[cfg(not(feature = "v10"))] +#[cfg(feature = "v9")] pub async fn create_feed(page_size: usize) -> Result, Error> { let create = |_store: Store| async move { Ok(ram::RandomAccessMemory::new(page_size)) }.boxed(); let storage = Storage::new(create, false).await?; diff --git a/tests/compat.rs b/tests/compat.rs index 41de7a78..1f8a2da4 100644 --- a/tests/compat.rs +++ b/tests/compat.rs @@ -11,14 +11,14 @@ use std::path::{Path, PathBuf}; use data_encoding::HEXLOWER; use ed25519_dalek::{Keypair, Signature}; -#[cfg(not(feature = "v10"))] +#[cfg(feature = "v9")] use hypercore::Feed; use hypercore::{Storage, Store}; use random_access_disk::RandomAccessDisk; use remove_dir_all::remove_dir_all; #[async_std::test] -#[cfg(not(feature = "v10"))] +#[cfg(feature = "v9")] async fn deterministic_data_and_tree() { let expected_tree = hex_bytes(concat!( "0502570200002807424c414b4532620000000000000000000000000000000000ab27d45f509274", @@ -59,7 +59,7 @@ fn deterministic_data_and_tree_after_replication() { } #[async_std::test] -#[cfg(not(feature = "v10"))] +#[cfg(feature = "v9")] async fn deterministic_signatures() { let key = hex_bytes("9718a1ff1c4ca79feac551c0c7212a65e4091278ec886b88be01ee4039682238"); let keypair_bytes = hex_bytes(concat!( @@ -138,7 +138,7 @@ fn hex_bytes(hex: &str) -> Vec { HEXLOWER.decode(hex.as_bytes()).unwrap() } -#[cfg(not(feature = "v10"))] +#[cfg(feature = "v9")] fn storage_path>(dir: P, s: Store) -> PathBuf { let filename = match s { Store::Tree => "tree", @@ -152,7 +152,7 @@ fn storage_path>(dir: P, s: Store) -> PathBuf { dir.as_ref().join(filename) } -#[cfg(not(feature = "v10"))] +#[cfg(feature = "v9")] async fn mk_storage() -> (PathBuf, Storage) { let temp_dir = tempfile::tempdir().unwrap(); let dir = temp_dir.into_path(); @@ -168,7 +168,7 @@ async fn mk_storage() -> (PathBuf, Storage) { (dir, storage) } -#[cfg(not(feature = "v10"))] +#[cfg(feature = "v9")] fn read_bytes>(dir: P, s: Store) -> Vec { let mut f = File::open(storage_path(dir, s)).unwrap(); let mut bytes = Vec::new(); @@ -176,7 +176,7 @@ fn read_bytes>(dir: P, s: Store) -> Vec { bytes } -#[cfg(not(feature = "v10"))] +#[cfg(feature = "v9")] fn mk_keypair(keypair_bytes: &[u8], public_key: &[u8]) -> Keypair { let keypair = Keypair::from_bytes(&keypair_bytes).unwrap(); assert_eq!( diff --git a/tests/feed.rs b/tests/feed.rs index 5952011f..4c1e158c 100644 --- a/tests/feed.rs +++ b/tests/feed.rs @@ -2,9 +2,9 @@ extern crate random_access_memory as ram; mod common; -#[cfg(not(feature = "v10"))] +#[cfg(feature = "v9")] use common::create_feed; -#[cfg(not(feature = "v10"))] +#[cfg(feature = "v9")] use hypercore::{generate_keypair, Feed, NodeTrait, PublicKey, SecretKey, Storage}; use random_access_storage::RandomAccess; use std::env::temp_dir; @@ -13,7 +13,7 @@ use std::fs; use std::io::Write; #[async_std::test] -#[cfg(not(feature = "v10"))] +#[cfg(feature = "v9")] async fn create_with_key() { let keypair = generate_keypair(); let storage = Storage::new_memory().await.unwrap(); @@ -25,7 +25,7 @@ async fn create_with_key() { } #[async_std::test] -#[cfg(not(feature = "v10"))] +#[cfg(feature = "v9")] async fn display() { let feed = create_feed(50).await.unwrap(); let output = format!("{}", feed); @@ -33,7 +33,7 @@ async fn display() { } #[async_std::test] -#[cfg(not(feature = "v10"))] +#[cfg(feature = "v9")] /// Verify `.append()` and `.get()` work. async fn set_get() { let mut feed = create_feed(50).await.unwrap(); @@ -45,7 +45,7 @@ async fn set_get() { } #[async_std::test] -#[cfg(not(feature = "v10"))] +#[cfg(feature = "v9")] async fn append() { let mut feed = create_feed(50).await.unwrap(); feed.append(br#"{"hello":"world"}"#).await.unwrap(); @@ -70,7 +70,7 @@ async fn append() { } #[async_std::test] -#[cfg(not(feature = "v10"))] +#[cfg(feature = "v9")] /// Verify the `.root_hashes()` method returns the right nodes. async fn root_hashes() { // If no roots exist we should get an error. @@ -98,7 +98,7 @@ async fn root_hashes() { } #[async_std::test] -#[cfg(not(feature = "v10"))] +#[cfg(feature = "v9")] async fn verify() { let mut feed = create_feed(50).await.unwrap(); let (public, secret) = copy_keys(&feed); @@ -135,7 +135,7 @@ async fn verify() { } #[async_std::test] -#[cfg(not(feature = "v10"))] +#[cfg(feature = "v9")] async fn put() { let mut a = create_feed(50).await.unwrap(); let (public, secret) = copy_keys(&a); @@ -165,7 +165,7 @@ async fn put() { } #[async_std::test] -#[cfg(not(feature = "v10"))] +#[cfg(feature = "v9")] /// Put data from one feed into another, while veryfing hashes. /// I.e. manual replication between two feeds. async fn put_with_data() { @@ -206,7 +206,7 @@ async fn put_with_data() { } #[async_std::test] -#[cfg(not(feature = "v10"))] +#[cfg(feature = "v9")] async fn create_with_storage() { let storage = Storage::new_memory().await.unwrap(); assert!( @@ -216,7 +216,7 @@ async fn create_with_storage() { } #[async_std::test] -#[cfg(not(feature = "v10"))] +#[cfg(feature = "v9")] async fn create_with_stored_public_key() { let mut storage = Storage::new_memory().await.unwrap(); let keypair = generate_keypair(); @@ -228,7 +228,7 @@ async fn create_with_stored_public_key() { } #[async_std::test] -#[cfg(not(feature = "v10"))] +#[cfg(feature = "v9")] async fn create_with_stored_keys() { let mut storage = Storage::new_memory().await.unwrap(); let keypair = generate_keypair(); @@ -240,7 +240,7 @@ async fn create_with_stored_keys() { ); } -#[cfg(not(feature = "v10"))] +#[cfg(feature = "v9")] fn copy_keys( feed: &Feed> + Debug + Send>, ) -> (PublicKey, SecretKey) { @@ -259,7 +259,7 @@ fn copy_keys( } #[async_std::test] -#[cfg(not(feature = "v10"))] +#[cfg(feature = "v9")] async fn audit() { let mut feed = create_feed(50).await.unwrap(); feed.append(b"hello").await.unwrap(); @@ -276,7 +276,7 @@ async fn audit() { } #[async_std::test] -#[cfg(not(feature = "v10"))] +#[cfg(feature = "v9")] async fn audit_bad_data() { let mut dir = temp_dir(); dir.push("audit_bad_data"); @@ -326,7 +326,7 @@ async fn audit_bad_data() { } #[async_std::test] -#[cfg(not(feature = "v10"))] +#[cfg(feature = "v9")] async fn try_open_missing_dir() { use rand::distributions::Alphanumeric; use rand::{thread_rng, Rng}; @@ -350,7 +350,7 @@ async fn try_open_missing_dir() { } #[async_std::test] -#[cfg(not(feature = "v10"))] +#[cfg(feature = "v9")] async fn try_open_file_as_dir() { if Feed::open("Cargo.toml").await.is_ok() { panic!("Opening path that points to a file must result in error"); diff --git a/tests/model.rs b/tests/model.rs index 27648b8e..bf43419f 100644 --- a/tests/model.rs +++ b/tests/model.rs @@ -1,6 +1,6 @@ mod common; -#[cfg(not(feature = "v10"))] +#[cfg(feature = "v9")] use common::create_feed; use quickcheck::{quickcheck, Arbitrary, Gen}; use rand::seq::SliceRandom; @@ -38,7 +38,7 @@ impl Arbitrary for Op { } } -#[cfg(not(feature = "v10"))] +#[cfg(feature = "v9")] quickcheck! { fn implementation_matches_model(ops: Vec) -> bool { async_std::task::block_on(async { diff --git a/tests/regression.rs b/tests/regression.rs index 61cf60ef..6c5fc882 100644 --- a/tests/regression.rs +++ b/tests/regression.rs @@ -1,6 +1,6 @@ mod common; -#[cfg(not(feature = "v10"))] +#[cfg(feature = "v9")] use common::create_feed; // Postmortem: errors were happening correctly, but the error check in @@ -8,7 +8,7 @@ use common::create_feed; // checking inclusively `<=`. All we had to do was fix the check, and we all // good. #[async_std::test] -#[cfg(not(feature = "v10"))] +#[cfg(feature = "v9")] async fn regression_01() { let mut feed = create_feed(50).await.unwrap(); assert_eq!(feed.len(), 0); diff --git a/tests/storage.rs b/tests/storage.rs index 834bab32..62bcf2e3 100644 --- a/tests/storage.rs +++ b/tests/storage.rs @@ -1,11 +1,11 @@ -#[cfg(not(feature = "v10"))] +#[cfg(feature = "v9")] use ed25519_dalek::PublicKey; -#[cfg(not(feature = "v10"))] +#[cfg(feature = "v9")] use hypercore::{generate_keypair, sign, verify, Signature, Storage}; #[async_std::test] -#[cfg(not(feature = "v10"))] +#[cfg(feature = "v9")] async fn should_write_and_read_keypair() { let keypair = generate_keypair(); let msg = b"hello"; @@ -29,7 +29,7 @@ async fn should_write_and_read_keypair() { } #[async_std::test] -#[cfg(not(feature = "v10"))] +#[cfg(feature = "v9")] async fn should_read_partial_keypair() { let keypair = generate_keypair(); let mut storage = Storage::new_memory().await.unwrap(); @@ -43,7 +43,7 @@ async fn should_read_partial_keypair() { } #[async_std::test] -#[cfg(not(feature = "v10"))] +#[cfg(feature = "v9")] async fn should_read_no_keypair() { let mut storage = Storage::new_memory().await.unwrap(); let partial = storage.read_partial_keypair().await; @@ -51,7 +51,7 @@ async fn should_read_no_keypair() { } #[async_std::test] -#[cfg(not(feature = "v10"))] +#[cfg(feature = "v9")] async fn should_read_empty_public_key() { let mut storage = Storage::new_memory().await.unwrap(); assert!(storage.read_public_key().await.is_err()); From 3fdfd78c5f0db6cb38b22c450ddd0a4132f4c725 Mon Sep 17 00:00:00 2001 From: Timo Tiuraniemi Date: Tue, 20 Sep 2022 13:33:47 +0300 Subject: [PATCH 065/157] For some reason from_bytes fails to compile when referenced in another crate --- src/core.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/core.rs b/src/core.rs index b35edd2b..42f6f107 100644 --- a/src/core.rs +++ b/src/core.rs @@ -13,6 +13,7 @@ use anyhow::{anyhow, Result}; use ed25519_dalek::Signature; use futures::future::Either; use random_access_storage::RandomAccess; +use std::convert::TryFrom; use std::fmt::Debug; /// Hypercore is an append-only log structure. @@ -163,7 +164,7 @@ where } }; changeset.ancestors = tree_upgrade.ancestors; - changeset.signature = Some(Signature::from_bytes(&tree_upgrade.signature)?); + changeset.signature = Some(Signature::try_from(&*tree_upgrade.signature)?); // TODO: Skip reorg hints for now, seems to only have to do with replication // addReorgHint(header.hints.reorgs, tree, batch) From 4780a59baf105b5a466a2320e77357853c59975d Mon Sep 17 00:00:00 2001 From: Timo Tiuraniemi Date: Wed, 21 Sep 2022 12:24:41 +0300 Subject: [PATCH 066/157] Provide method to access the keypair --- src/core.rs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/core.rs b/src/core.rs index 42f6f107..34da8c8b 100644 --- a/src/core.rs +++ b/src/core.rs @@ -371,6 +371,11 @@ where Ok(()) } + /// Access the key pair. + pub fn key_pair(&self) -> &PartialKeypair { + &self.key_pair + } + fn should_flush_bitfield_and_tree_and_oplog(&mut self) -> bool { if self.skip_flush_count == 0 || self.oplog.entries_byte_length >= MAX_OPLOG_ENTRIES_BYTE_SIZE From 37a7eb96232edd0a4042c04b569bef9e00fe786f Mon Sep 17 00:00:00 2001 From: Timo Tiuraniemi Date: Mon, 10 Oct 2022 10:16:20 +0300 Subject: [PATCH 067/157] Encode Vec buffer, move replicator stuff out to hypercore-protocol-rs --- src/compact_encoding/mod.rs | 32 ++++++++++++++++++++++++++++---- src/crypto/hash.rs | 13 +------------ src/lib.rs | 5 ++++- 3 files changed, 33 insertions(+), 17 deletions(-) diff --git a/src/compact_encoding/mod.rs b/src/compact_encoding/mod.rs index 10a86aa5..2eae16b8 100644 --- a/src/compact_encoding/mod.rs +++ b/src/compact_encoding/mod.rs @@ -8,7 +8,7 @@ const U32_SIGNIFIER: u8 = 0xfe; const U64_SIGNIFIER: u8 = 0xff; /// State. -#[derive(Debug)] +#[derive(Debug, Clone)] pub struct State { /// Start position pub start: usize, @@ -205,6 +205,13 @@ impl State { self.end += len; } + /// Preencode a vector byte buffer + pub fn preencode_buffer_vec(&mut self, value: &Vec) { + let len = value.len(); + self.preencode_usize_var(&len); + self.end += len; + } + /// Encode a byte buffer pub fn encode_buffer(&mut self, value: &[u8], buffer: &mut [u8]) { let len = value.len(); @@ -215,10 +222,13 @@ impl State { /// Decode a byte buffer pub fn decode_buffer(&mut self, buffer: &[u8]) -> Box<[u8]> { + self.decode_buffer_vec(buffer).into_boxed_slice() + } + + /// Decode a vector byte buffer + pub fn decode_buffer_vec(&mut self, buffer: &[u8]) -> Vec { let len = self.decode_usize_var(buffer); - let value = buffer[self.start..self.start + len] - .to_vec() - .into_boxed_slice(); + let value = buffer[self.start..self.start + len].to_vec(); self.start += value.len(); value } @@ -430,6 +440,20 @@ impl CompactEncoding> for State { } } +impl CompactEncoding> for State { + fn preencode(&mut self, value: &Vec) { + self.preencode_buffer_vec(value); + } + + fn encode(&mut self, value: &Vec, buffer: &mut [u8]) { + self.encode_buffer(value, buffer); + } + + fn decode(&mut self, buffer: &[u8]) -> Vec { + self.decode_buffer_vec(buffer).to_vec() + } +} + impl CompactEncoding> for State { fn preencode(&mut self, value: &Vec) { self.preencode_string_array(value); diff --git a/src/crypto/hash.rs b/src/crypto/hash.rs index b5b458d4..c33417e9 100644 --- a/src/crypto/hash.rs +++ b/src/crypto/hash.rs @@ -22,14 +22,6 @@ const TREE: [u8; 32] = [ 0x9F, 0xAC, 0x70, 0xB5, 0xC, 0xA1, 0x4E, 0xFC, 0x4E, 0x91, 0xC8, 0x33, 0xB2, 0x4, 0xE7, 0x5B, 0x8B, 0x5A, 0xAD, 0x8B, 0x58, 0x81, 0xBF, 0xC0, 0xAD, 0xB5, 0xEF, 0x38, 0xA3, 0x27, 0x5B, 0x9C, ]; -const REPLICATE_INITIATOR: [u8; 32] = [ - 0x51, 0x81, 0x2A, 0x2A, 0x35, 0x9B, 0x50, 0x36, 0x95, 0x36, 0x77, 0x5D, 0xF8, 0x9E, 0x18, 0xE4, - 0x77, 0x40, 0xF3, 0xDB, 0x72, 0xAC, 0xA, 0xE7, 0xB, 0x29, 0x59, 0x4C, 0x19, 0x4D, 0xC3, 0x16, -]; -const REPLICATE_RESPONDER: [u8; 32] = [ - 0x4, 0x38, 0x49, 0x2D, 0x2, 0x97, 0xC, 0xC1, 0x35, 0x28, 0xAC, 0x2, 0x62, 0xBC, 0xA0, 0x7, - 0x4E, 0x9, 0x26, 0x26, 0x2, 0x56, 0x86, 0x5A, 0xCC, 0xC0, 0xBF, 0x15, 0xBD, 0x79, 0x12, 0x7D, -]; /// `BLAKE2b` hash. #[derive(Debug, Clone, PartialEq)] @@ -190,6 +182,7 @@ impl DerefMut for Hash { } /// Create a signable buffer for tree. This is treeSignable in Javascript. +/// See https://github.com/hypercore-protocol/hypercore/blob/70b271643c4e4b1e5ecae5bb579966dfe6361ff3/lib/caps.js#L17 pub fn signable_tree(hash: &[u8], length: u64, fork: u64) -> Box<[u8]> { let (mut state, mut buffer) = State::new_with_size(80); state.encode_fixed_32(&TREE, &mut buffer); @@ -329,9 +322,5 @@ mod tests { let ns = hash.as_bytes(); let tree: Box<[u8]> = { hash_with_extra_byte(ns, 0).into() }; assert_eq!(tree, TREE.into()); - let replicate_initiator: Box<[u8]> = { hash_with_extra_byte(ns, 1).into() }; - assert_eq!(replicate_initiator, REPLICATE_INITIATOR.into()); - let replicate_responder: Box<[u8]> = { hash_with_extra_byte(ns, 2).into() }; - assert_eq!(replicate_responder, REPLICATE_RESPONDER.into()); } } diff --git a/src/lib.rs b/src/lib.rs index 9d5bdb91..189bab77 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -87,7 +87,10 @@ pub use crate::replicate::Peer; pub use crate::storage::{NodeTrait, PartialKeypair, Storage, Store}; #[cfg(feature = "v10")] pub use crate::storage_v10::{PartialKeypair, Storage}; -pub use ed25519_dalek::{PublicKey, SecretKey}; +pub use ed25519_dalek::{ + ExpandedSecretKey, Keypair, PublicKey, SecretKey, EXPANDED_SECRET_KEY_LENGTH, KEYPAIR_LENGTH, + PUBLIC_KEY_LENGTH, SECRET_KEY_LENGTH, +}; use std::path::Path; From 2ac5765f8f7770fbf4b4cbe65ff6f4d1a2c083a6 Mon Sep 17 00:00:00 2001 From: Timo Tiuraniemi Date: Sat, 15 Oct 2022 10:37:53 +0300 Subject: [PATCH 068/157] Move Node compact encoding to the main module to use in protocol --- src/compact_encoding/mod.rs | 66 +++++++++++++++++++++++++++++++++++++ src/core.rs | 2 ++ src/oplog/entry.rs | 48 --------------------------- 3 files changed, 68 insertions(+), 48 deletions(-) diff --git a/src/compact_encoding/mod.rs b/src/compact_encoding/mod.rs index 2eae16b8..767fb635 100644 --- a/src/compact_encoding/mod.rs +++ b/src/compact_encoding/mod.rs @@ -1,5 +1,6 @@ //! Compact encoding module. Rust implementation of https://github.com/compact-encoding/compact-encoding. +use crate::Node; use std::convert::TryFrom; use std::fmt::Debug; @@ -384,6 +385,23 @@ impl CompactEncoding for State { } } +impl CompactEncoding for State { + fn preencode(&mut self, _: &u8) { + self.end += 1; + } + + fn encode(&mut self, value: &u8, buffer: &mut [u8]) { + buffer[self.start] = *value; + self.start += 1; + } + + fn decode(&mut self, buffer: &[u8]) -> u8 { + let value = buffer[self.start]; + self.start += 1; + value + } +} + impl CompactEncoding for State { fn preencode(&mut self, value: &u32) { self.preencode_uint_var(value) @@ -467,3 +485,51 @@ impl CompactEncoding> for State { self.decode_string_array(buffer) } } + +impl CompactEncoding for State { + fn preencode(&mut self, value: &Node) { + self.preencode(&value.index); + self.preencode(&value.length); + self.preencode_fixed_32(); + } + + fn encode(&mut self, value: &Node, buffer: &mut [u8]) { + self.encode(&value.index, buffer); + self.encode(&value.length, buffer); + self.encode_fixed_32(&value.hash, buffer); + } + + fn decode(&mut self, buffer: &[u8]) -> Node { + let index: u64 = self.decode(buffer); + let length: u64 = self.decode(buffer); + let hash: Box<[u8]> = self.decode_fixed_32(buffer); + Node::new(index, hash.to_vec(), length) + } +} + +impl CompactEncoding> for State { + fn preencode(&mut self, value: &Vec) { + let len = value.len(); + self.preencode(&len); + for val in value.into_iter() { + self.preencode(val); + } + } + + fn encode(&mut self, value: &Vec, buffer: &mut [u8]) { + let len = value.len(); + self.encode(&len, buffer); + for val in value { + self.encode(val, buffer); + } + } + + fn decode(&mut self, buffer: &[u8]) -> Vec { + let len: usize = self.decode(buffer); + let mut value = Vec::with_capacity(len); + for _ in 0..len { + value.push(self.decode(buffer)); + } + value + } +} diff --git a/src/core.rs b/src/core.rs index 34da8c8b..70045071 100644 --- a/src/core.rs +++ b/src/core.rs @@ -45,6 +45,7 @@ pub struct Info { pub length: u64, pub byte_length: u64, pub contiguous_length: u64, + pub fork: u64, } impl Hypercore @@ -193,6 +194,7 @@ where length: self.tree.length, byte_length: self.tree.byte_length, contiguous_length: self.header.contiguous_length, + fork: self.tree.fork, } } diff --git a/src/oplog/entry.rs b/src/oplog/entry.rs index 2aa09674..05f1f815 100644 --- a/src/oplog/entry.rs +++ b/src/oplog/entry.rs @@ -3,54 +3,6 @@ use crate::{ Node, }; -impl CompactEncoding for State { - fn preencode(&mut self, value: &Node) { - self.preencode(&value.index); - self.preencode(&value.length); - self.preencode_fixed_32(); - } - - fn encode(&mut self, value: &Node, buffer: &mut [u8]) { - self.encode(&value.index, buffer); - self.encode(&value.length, buffer); - self.encode_fixed_32(&value.hash, buffer); - } - - fn decode(&mut self, buffer: &[u8]) -> Node { - let index: u64 = self.decode(buffer); - let length: u64 = self.decode(buffer); - let hash: Box<[u8]> = self.decode_fixed_32(buffer); - Node::new(index, hash.to_vec(), length) - } -} - -impl CompactEncoding> for State { - fn preencode(&mut self, value: &Vec) { - let len = value.len(); - self.preencode(&len); - for val in value.into_iter() { - self.preencode(val); - } - } - - fn encode(&mut self, value: &Vec, buffer: &mut [u8]) { - let len = value.len(); - self.encode(&len, buffer); - for val in value { - self.encode(val, buffer); - } - } - - fn decode(&mut self, buffer: &[u8]) -> Vec { - let len: usize = self.decode(buffer); - let mut value = Vec::with_capacity(len); - for _ in 0..len { - value.push(self.decode(buffer)); - } - value - } -} - /// Entry tree upgrade #[derive(Debug)] pub struct EntryTreeUpgrade { From 821e7563f4f05bce4a54df89cd157b5b62902504 Mon Sep 17 00:00:00 2001 From: Timo Tiuraniemi Date: Mon, 17 Oct 2022 09:55:49 +0300 Subject: [PATCH 069/157] Reorganize compact_encoding. Add types needed for verifying remote data. --- src/common/data.rs | 45 +++ src/common/mod.rs | 2 + src/compact_encoding/custom.rs | 145 +++++++++ src/compact_encoding/generic.rs | 117 +++++++ src/compact_encoding/mod.rs | 536 +------------------------------- src/compact_encoding/types.rs | 376 ++++++++++++++++++++++ src/lib.rs | 2 +- 7 files changed, 690 insertions(+), 533 deletions(-) create mode 100644 src/common/data.rs create mode 100644 src/compact_encoding/custom.rs create mode 100644 src/compact_encoding/generic.rs create mode 100644 src/compact_encoding/types.rs diff --git a/src/common/data.rs b/src/common/data.rs new file mode 100644 index 00000000..2afda2fe --- /dev/null +++ b/src/common/data.rs @@ -0,0 +1,45 @@ +use crate::Node; + +#[derive(Debug, Clone, PartialEq)] +/// Block of data +pub struct DataBlock { + /// Hypercore index + pub index: u64, + /// Data block value in bytes + pub value: Vec, + /// TODO: document + pub nodes: Vec, +} + +#[derive(Debug, Clone, PartialEq)] +/// Data hash +pub struct DataHash { + /// Hypercore index + pub index: u64, + /// TODO: document + pub nodes: Vec, +} + +#[derive(Debug, Clone, PartialEq)] +/// TODO: Document +pub struct DataSeek { + /// TODO: Document + pub bytes: u64, + /// TODO: Document + pub nodes: Vec, +} + +#[derive(Debug, Clone, PartialEq)] +/// TODO: Document +pub struct DataUpgrade { + /// TODO: Document + pub start: u64, + /// TODO: Document + pub length: u64, + /// TODO: Document + pub nodes: Vec, + /// TODO: Document + pub additional_nodes: Vec, + /// TODO: Document + pub signature: Vec, +} diff --git a/src/common/mod.rs b/src/common/mod.rs index 4a6c6459..8a8f8cb4 100644 --- a/src/common/mod.rs +++ b/src/common/mod.rs @@ -1,5 +1,7 @@ +mod data; mod node; mod store; +pub use self::data::{DataBlock, DataHash, DataSeek, DataUpgrade}; pub use self::node::{Node, NodeByteRange}; pub use self::store::{Store, StoreInfo, StoreInfoInstruction, StoreInfoType}; diff --git a/src/compact_encoding/custom.rs b/src/compact_encoding/custom.rs new file mode 100644 index 00000000..46fced17 --- /dev/null +++ b/src/compact_encoding/custom.rs @@ -0,0 +1,145 @@ +//! Hypercore-specific compact encodings +use super::{CompactEncoding, State}; +use crate::{DataBlock, DataHash, DataSeek, DataUpgrade, Node}; + +impl CompactEncoding for State { + fn preencode(&mut self, value: &Node) { + self.preencode(&value.index); + self.preencode(&value.length); + self.preencode_fixed_32(); + } + + fn encode(&mut self, value: &Node, buffer: &mut [u8]) { + self.encode(&value.index, buffer); + self.encode(&value.length, buffer); + self.encode_fixed_32(&value.hash, buffer); + } + + fn decode(&mut self, buffer: &[u8]) -> Node { + let index: u64 = self.decode(buffer); + let length: u64 = self.decode(buffer); + let hash: Box<[u8]> = self.decode_fixed_32(buffer); + Node::new(index, hash.to_vec(), length) + } +} + +impl CompactEncoding> for State { + fn preencode(&mut self, value: &Vec) { + let len = value.len(); + self.preencode(&len); + for val in value.into_iter() { + self.preencode(val); + } + } + + fn encode(&mut self, value: &Vec, buffer: &mut [u8]) { + let len = value.len(); + self.encode(&len, buffer); + for val in value { + self.encode(val, buffer); + } + } + + fn decode(&mut self, buffer: &[u8]) -> Vec { + let len: usize = self.decode(buffer); + let mut value = Vec::with_capacity(len); + for _ in 0..len { + value.push(self.decode(buffer)); + } + value + } +} + +impl CompactEncoding for State { + fn preencode(&mut self, value: &DataBlock) { + self.preencode(&value.index); + self.preencode(&value.value); + self.preencode(&value.nodes); + } + + fn encode(&mut self, value: &DataBlock, buffer: &mut [u8]) { + self.encode(&value.index, buffer); + self.encode(&value.value, buffer); + self.encode(&value.nodes, buffer); + } + + fn decode(&mut self, buffer: &[u8]) -> DataBlock { + let index: u64 = self.decode(buffer); + let value: Vec = self.decode(buffer); + let nodes: Vec = self.decode(buffer); + DataBlock { + index, + value, + nodes, + } + } +} + +impl CompactEncoding for State { + fn preencode(&mut self, value: &DataHash) { + self.preencode(&value.index); + self.preencode(&value.nodes); + } + + fn encode(&mut self, value: &DataHash, buffer: &mut [u8]) { + self.encode(&value.index, buffer); + self.encode(&value.nodes, buffer); + } + + fn decode(&mut self, buffer: &[u8]) -> DataHash { + let index: u64 = self.decode(buffer); + let nodes: Vec = self.decode(buffer); + DataHash { index, nodes } + } +} + +impl CompactEncoding for State { + fn preencode(&mut self, value: &DataSeek) { + self.preencode(&value.bytes); + self.preencode(&value.nodes); + } + + fn encode(&mut self, value: &DataSeek, buffer: &mut [u8]) { + self.encode(&value.bytes, buffer); + self.encode(&value.nodes, buffer); + } + + fn decode(&mut self, buffer: &[u8]) -> DataSeek { + let bytes: u64 = self.decode(buffer); + let nodes: Vec = self.decode(buffer); + DataSeek { bytes, nodes } + } +} + +impl CompactEncoding for State { + fn preencode(&mut self, value: &DataUpgrade) { + self.preencode(&value.start); + self.preencode(&value.length); + self.preencode(&value.nodes); + self.preencode(&value.additional_nodes); + self.preencode(&value.signature); + } + + fn encode(&mut self, value: &DataUpgrade, buffer: &mut [u8]) { + self.encode(&value.start, buffer); + self.encode(&value.length, buffer); + self.encode(&value.nodes, buffer); + self.encode(&value.additional_nodes, buffer); + self.encode(&value.signature, buffer); + } + + fn decode(&mut self, buffer: &[u8]) -> DataUpgrade { + let start: u64 = self.decode(buffer); + let length: u64 = self.decode(buffer); + let nodes: Vec = self.decode(buffer); + let additional_nodes: Vec = self.decode(buffer); + let signature: Vec = self.decode(buffer); + DataUpgrade { + start, + length, + nodes, + additional_nodes, + signature, + } + } +} diff --git a/src/compact_encoding/generic.rs b/src/compact_encoding/generic.rs new file mode 100644 index 00000000..809903c0 --- /dev/null +++ b/src/compact_encoding/generic.rs @@ -0,0 +1,117 @@ +//! Generic compact encodings +use super::{CompactEncoding, State}; + +impl CompactEncoding for State { + fn preencode(&mut self, value: &String) { + self.preencode_str(value) + } + + fn encode(&mut self, value: &String, buffer: &mut [u8]) { + self.encode_str(value, buffer) + } + + fn decode(&mut self, buffer: &[u8]) -> String { + self.decode_string(buffer) + } +} + +impl CompactEncoding for State { + fn preencode(&mut self, _: &u8) { + self.end += 1; + } + + fn encode(&mut self, value: &u8, buffer: &mut [u8]) { + buffer[self.start] = *value; + self.start += 1; + } + + fn decode(&mut self, buffer: &[u8]) -> u8 { + let value = buffer[self.start]; + self.start += 1; + value + } +} + +impl CompactEncoding for State { + fn preencode(&mut self, value: &u32) { + self.preencode_uint_var(value) + } + + fn encode(&mut self, value: &u32, buffer: &mut [u8]) { + self.encode_u32_var(value, buffer) + } + + fn decode(&mut self, buffer: &[u8]) -> u32 { + self.decode_u32_var(buffer) + } +} + +impl CompactEncoding for State { + fn preencode(&mut self, value: &u64) { + self.preencode_uint_var(value) + } + + fn encode(&mut self, value: &u64, buffer: &mut [u8]) { + self.encode_u64_var(value, buffer) + } + + fn decode(&mut self, buffer: &[u8]) -> u64 { + self.decode_u64_var(buffer) + } +} + +impl CompactEncoding for State { + fn preencode(&mut self, value: &usize) { + self.preencode_usize_var(value) + } + + fn encode(&mut self, value: &usize, buffer: &mut [u8]) { + self.encode_usize_var(value, buffer) + } + + fn decode(&mut self, buffer: &[u8]) -> usize { + self.decode_usize_var(buffer) + } +} + +impl CompactEncoding> for State { + fn preencode(&mut self, value: &Box<[u8]>) { + self.preencode_buffer(value); + } + + fn encode(&mut self, value: &Box<[u8]>, buffer: &mut [u8]) { + self.encode_buffer(value, buffer); + } + + fn decode(&mut self, buffer: &[u8]) -> Box<[u8]> { + self.decode_buffer(buffer) + } +} + +impl CompactEncoding> for State { + fn preencode(&mut self, value: &Vec) { + self.preencode_buffer_vec(value); + } + + fn encode(&mut self, value: &Vec, buffer: &mut [u8]) { + self.encode_buffer(value, buffer); + } + + fn decode(&mut self, buffer: &[u8]) -> Vec { + self.decode_buffer_vec(buffer).to_vec() + } +} + +impl CompactEncoding> for State { + fn preencode(&mut self, value: &Vec) { + self.preencode_string_array(value); + } + + fn encode(&mut self, value: &Vec, buffer: &mut [u8]) { + self.encode_string_array(value, buffer); + } + + fn decode(&mut self, buffer: &[u8]) -> Vec { + self.decode_string_array(buffer) + } +} diff --git a/src/compact_encoding/mod.rs b/src/compact_encoding/mod.rs index 767fb635..3a75d5c5 100644 --- a/src/compact_encoding/mod.rs +++ b/src/compact_encoding/mod.rs @@ -1,535 +1,7 @@ //! Compact encoding module. Rust implementation of https://github.com/compact-encoding/compact-encoding. -use crate::Node; -use std::convert::TryFrom; -use std::fmt::Debug; +pub mod custom; +pub mod generic; +pub mod types; -const U16_SIGNIFIER: u8 = 0xfd; -const U32_SIGNIFIER: u8 = 0xfe; -const U64_SIGNIFIER: u8 = 0xff; - -/// State. -#[derive(Debug, Clone)] -pub struct State { - /// Start position - pub start: usize, - /// End position - pub end: usize, -} - -impl State { - /// Create emtpy state - pub fn new() -> State { - State::new_with_start_and_end(0, 0) - } - - /// Create a state with an already known size. - /// With this, you can/must skip the preencode step. - pub fn new_with_size(size: usize) -> (State, Box<[u8]>) { - ( - State::new_with_start_and_end(0, size), - vec![0; size].into_boxed_slice(), - ) - } - - /// Create a state with a start and end already known. - pub fn new_with_start_and_end(start: usize, end: usize) -> State { - State { start, end } - } - - /// Create a state from existing buffer. - pub fn from_buffer(buffer: &[u8]) -> State { - State::new_with_start_and_end(0, buffer.len()) - } - - /// After calling preencode(), this allocates the right size buffer to the heap. - /// Follow this with the same number of encode() steps to fill the created buffer. - pub fn create_buffer(&self) -> Box<[u8]> { - vec![0; self.end].into_boxed_slice() - } - - /// Preencode a string slice - pub fn preencode_str(&mut self, value: &str) { - self.preencode_usize_var(&value.len()); - self.end += value.len(); - } - - /// Encode a string slice - pub fn encode_str(&mut self, value: &str, buffer: &mut [u8]) { - let len = value.len(); - self.encode_usize_var(&len, buffer); - buffer[self.start..self.start + len].copy_from_slice(value.as_bytes()); - self.start += len; - } - - /// Decode a String - pub fn decode_string(&mut self, buffer: &[u8]) -> String { - let len = self.decode_usize_var(buffer); - let value = std::str::from_utf8(&buffer[self.start..self.start + len]) - .expect("string is invalid UTF-8"); - self.start += len; - value.to_string() - } - - /// Preencode a variable length usigned int - pub fn preencode_uint_var + Ord>(&mut self, uint: &T) { - self.end += if *uint < T::from(U16_SIGNIFIER.into()) { - 1 - } else if *uint <= T::from(0xffff) { - 3 - } else if *uint <= T::from(0xffffffff) { - 5 - } else { - 9 - }; - } - - /// Decode a fixed length u8 - pub fn decode_u8(&mut self, buffer: &[u8]) -> u8 { - let value: u8 = buffer[self.start]; - self.start += 1; - value - } - - /// Decode a fixed length u16 - pub fn decode_u16(&mut self, buffer: &[u8]) -> u16 { - let value: u16 = - ((buffer[self.start] as u16) << 0) | ((buffer[self.start + 1] as u16) << 8); - self.start += 2; - value - } - - /// Encode a variable length u32 - pub fn encode_u32_var(&mut self, value: &u32, buffer: &mut [u8]) { - if *value < U16_SIGNIFIER.into() { - let bytes = value.to_le_bytes(); - buffer[self.start] = bytes[0]; - self.start += 1; - } else if *value <= 0xffff { - buffer[self.start] = U16_SIGNIFIER; - self.start += 1; - self.encode_uint16_bytes(&value.to_le_bytes(), buffer); - } else { - buffer[self.start] = U32_SIGNIFIER; - self.start += 1; - self.encode_u32(*value, buffer); - } - } - - /// Encode u32 to 4 LE bytes. - pub fn encode_u32(&mut self, uint: u32, buffer: &mut [u8]) { - self.encode_uint32_bytes(&uint.to_le_bytes(), buffer); - } - - /// Decode a variable length u32 - pub fn decode_u32_var(&mut self, buffer: &[u8]) -> u32 { - let first = buffer[self.start]; - self.start += 1; - if first < U16_SIGNIFIER { - first.into() - } else if first == U16_SIGNIFIER { - self.decode_u16(buffer).into() - } else { - self.decode_u32(buffer).into() - } - } - - /// Decode a fixed length u32 - pub fn decode_u32(&mut self, buffer: &[u8]) -> u32 { - let value: u32 = ((buffer[self.start] as u32) << 0) - | ((buffer[self.start + 1] as u32) << 8) - | ((buffer[self.start + 2] as u32) << 16) - | ((buffer[self.start + 3] as u32) << 24); - self.start += 4; - value - } - - /// Encode a variable length u64 - pub fn encode_u64_var(&mut self, value: &u64, buffer: &mut [u8]) { - if *value < U16_SIGNIFIER.into() { - let bytes = value.to_le_bytes(); - buffer[self.start] = bytes[0]; - self.start += 1; - } else if *value <= 0xffff { - buffer[self.start] = U16_SIGNIFIER; - self.start += 1; - self.encode_uint16_bytes(&value.to_le_bytes(), buffer); - } else if *value <= 0xffffffff { - buffer[self.start] = U32_SIGNIFIER; - self.start += 1; - self.encode_uint32_bytes(&value.to_le_bytes(), buffer); - } else { - buffer[self.start] = U64_SIGNIFIER; - self.start += 1; - self.encode_uint64_bytes(&value.to_le_bytes(), buffer); - } - } - - /// Encode u64 to 8 LE bytes. - pub fn encode_u64(&mut self, uint: u64, buffer: &mut [u8]) { - self.encode_uint64_bytes(&uint.to_le_bytes(), buffer); - } - - /// Decode a variable length u64 - pub fn decode_u64_var(&mut self, buffer: &[u8]) -> u64 { - let first = buffer[self.start]; - self.start += 1; - if first < U16_SIGNIFIER { - first.into() - } else if first == U16_SIGNIFIER { - self.decode_u16(buffer).into() - } else if first == U32_SIGNIFIER { - self.decode_u32(buffer).into() - } else { - self.decode_u64(buffer) - } - } - - /// Decode a fixed length u64 - pub fn decode_u64(&mut self, buffer: &[u8]) -> u64 { - let value: u64 = ((buffer[self.start] as u64) << 0) - | ((buffer[self.start + 1] as u64) << 8) - | ((buffer[self.start + 2] as u64) << 16) - | ((buffer[self.start + 3] as u64) << 24) - | ((buffer[self.start + 4] as u64) << 32) - | ((buffer[self.start + 5] as u64) << 40) - | ((buffer[self.start + 6] as u64) << 48) - | ((buffer[self.start + 7] as u64) << 56); - self.start += 8; - value - } - - /// Preencode a byte buffer - pub fn preencode_buffer(&mut self, value: &Box<[u8]>) { - let len = value.len(); - self.preencode_usize_var(&len); - self.end += len; - } - - /// Preencode a vector byte buffer - pub fn preencode_buffer_vec(&mut self, value: &Vec) { - let len = value.len(); - self.preencode_usize_var(&len); - self.end += len; - } - - /// Encode a byte buffer - pub fn encode_buffer(&mut self, value: &[u8], buffer: &mut [u8]) { - let len = value.len(); - self.encode_usize_var(&len, buffer); - buffer[self.start..self.start + len].copy_from_slice(value); - self.start += len; - } - - /// Decode a byte buffer - pub fn decode_buffer(&mut self, buffer: &[u8]) -> Box<[u8]> { - self.decode_buffer_vec(buffer).into_boxed_slice() - } - - /// Decode a vector byte buffer - pub fn decode_buffer_vec(&mut self, buffer: &[u8]) -> Vec { - let len = self.decode_usize_var(buffer); - let value = buffer[self.start..self.start + len].to_vec(); - self.start += value.len(); - value - } - - /// Preencode a fixed 32 byte buffer - pub fn preencode_fixed_32(&mut self) { - self.end += 32; - } - - /// Encode a fixed 32 byte buffer - pub fn encode_fixed_32(&mut self, value: &[u8], buffer: &mut [u8]) { - buffer[self.start..self.start + 32].copy_from_slice(value); - self.start += 32; - } - - /// Encode a fixed 32 byte buffer - pub fn decode_fixed_32(&mut self, buffer: &[u8]) -> Box<[u8]> { - let value = buffer[self.start..self.start + 32] - .to_vec() - .into_boxed_slice(); - self.start += 32; - value - } - - /// Preencode a string array - pub fn preencode_string_array(&mut self, value: &Vec) { - let len = value.len(); - self.preencode_usize_var(&len); - for string_value in value.into_iter() { - self.preencode_str(string_value); - } - } - - /// Encode a String array - pub fn encode_string_array(&mut self, value: &Vec, buffer: &mut [u8]) { - let len = value.len(); - self.encode_usize_var(&len, buffer); - for string_value in value { - self.encode_str(string_value, buffer); - } - } - - /// Decode a String array - pub fn decode_string_array(&mut self, buffer: &[u8]) -> Vec { - let len = self.decode_usize_var(buffer); - let mut value = Vec::with_capacity(len); - for _ in 0..len { - value.push(self.decode_string(buffer)); - } - value - } - - fn encode_uint16_bytes(&mut self, bytes: &[u8], buffer: &mut [u8]) { - buffer[self.start] = bytes[0]; - buffer[self.start + 1] = bytes[1]; - self.start += 2; - } - - fn encode_uint32_bytes(&mut self, bytes: &[u8], buffer: &mut [u8]) { - self.encode_uint16_bytes(bytes, buffer); - buffer[self.start] = bytes[2]; - buffer[self.start + 1] = bytes[3]; - self.start += 2; - } - - fn encode_uint64_bytes(&mut self, bytes: &[u8], buffer: &mut [u8]) { - self.encode_uint32_bytes(bytes, buffer); - buffer[self.start] = bytes[4]; - buffer[self.start + 1] = bytes[5]; - buffer[self.start + 2] = bytes[6]; - buffer[self.start + 3] = bytes[7]; - self.start += 4; - } - - fn preencode_usize_var(&mut self, value: &usize) { - // TODO: This repeats the logic from above that works for u8 -> u64, but sadly not usize - self.end += if *value < U16_SIGNIFIER.into() { - 1 - } else if *value <= 0xffff { - 3 - } else if *value <= 0xffffffff { - 5 - } else { - 9 - }; - } - - fn encode_usize_var(&mut self, value: &usize, buffer: &mut [u8]) { - if *value <= 0xfc { - let bytes = value.to_le_bytes(); - buffer[self.start] = bytes[0]; - self.start += 1; - } else if *value <= 0xffff { - buffer[self.start] = U16_SIGNIFIER; - self.start += 1; - self.encode_uint16_bytes(&value.to_le_bytes(), buffer); - } else if *value <= 0xffffffff { - buffer[self.start] = U32_SIGNIFIER; - self.start += 1; - self.encode_uint32_bytes(&value.to_le_bytes(), buffer); - } else { - buffer[self.start] = U64_SIGNIFIER; - self.start += 1; - self.encode_uint64_bytes(&value.to_le_bytes(), buffer); - } - } - - fn decode_usize_var(&mut self, buffer: &[u8]) -> usize { - let first = buffer[self.start]; - self.start += 1; - // NB: the from_le_bytes needs a [u8; 2] and that can't be efficiently - // created from a byte slice. - if first < U16_SIGNIFIER { - first.into() - } else if first == U16_SIGNIFIER { - self.decode_u16(buffer).into() - } else if first == U32_SIGNIFIER { - usize::try_from(self.decode_u32(buffer)) - .expect("Attempted converting to a 32 bit usize on below 32 bit system") - } else { - usize::try_from(self.decode_u64(buffer)) - .expect("Attempted converting to a 64 bit usize on below 64 bit system") - } - } -} - -/// Compact Encoding -pub trait CompactEncoding -where - T: Debug, -{ - /// Preencode - fn preencode(&mut self, value: &T); - - /// Encode - fn encode(&mut self, value: &T, buffer: &mut [u8]); - - /// Decode - fn decode(&mut self, buffer: &[u8]) -> T; -} - -impl CompactEncoding for State { - fn preencode(&mut self, value: &String) { - self.preencode_str(value) - } - - fn encode(&mut self, value: &String, buffer: &mut [u8]) { - self.encode_str(value, buffer) - } - - fn decode(&mut self, buffer: &[u8]) -> String { - self.decode_string(buffer) - } -} - -impl CompactEncoding for State { - fn preencode(&mut self, _: &u8) { - self.end += 1; - } - - fn encode(&mut self, value: &u8, buffer: &mut [u8]) { - buffer[self.start] = *value; - self.start += 1; - } - - fn decode(&mut self, buffer: &[u8]) -> u8 { - let value = buffer[self.start]; - self.start += 1; - value - } -} - -impl CompactEncoding for State { - fn preencode(&mut self, value: &u32) { - self.preencode_uint_var(value) - } - - fn encode(&mut self, value: &u32, buffer: &mut [u8]) { - self.encode_u32_var(value, buffer) - } - - fn decode(&mut self, buffer: &[u8]) -> u32 { - self.decode_u32_var(buffer) - } -} - -impl CompactEncoding for State { - fn preencode(&mut self, value: &u64) { - self.preencode_uint_var(value) - } - - fn encode(&mut self, value: &u64, buffer: &mut [u8]) { - self.encode_u64_var(value, buffer) - } - - fn decode(&mut self, buffer: &[u8]) -> u64 { - self.decode_u64_var(buffer) - } -} - -impl CompactEncoding for State { - fn preencode(&mut self, value: &usize) { - self.preencode_usize_var(value) - } - - fn encode(&mut self, value: &usize, buffer: &mut [u8]) { - self.encode_usize_var(value, buffer) - } - - fn decode(&mut self, buffer: &[u8]) -> usize { - self.decode_usize_var(buffer) - } -} - -impl CompactEncoding> for State { - fn preencode(&mut self, value: &Box<[u8]>) { - self.preencode_buffer(value); - } - - fn encode(&mut self, value: &Box<[u8]>, buffer: &mut [u8]) { - self.encode_buffer(value, buffer); - } - - fn decode(&mut self, buffer: &[u8]) -> Box<[u8]> { - self.decode_buffer(buffer) - } -} - -impl CompactEncoding> for State { - fn preencode(&mut self, value: &Vec) { - self.preencode_buffer_vec(value); - } - - fn encode(&mut self, value: &Vec, buffer: &mut [u8]) { - self.encode_buffer(value, buffer); - } - - fn decode(&mut self, buffer: &[u8]) -> Vec { - self.decode_buffer_vec(buffer).to_vec() - } -} - -impl CompactEncoding> for State { - fn preencode(&mut self, value: &Vec) { - self.preencode_string_array(value); - } - - fn encode(&mut self, value: &Vec, buffer: &mut [u8]) { - self.encode_string_array(value, buffer); - } - - fn decode(&mut self, buffer: &[u8]) -> Vec { - self.decode_string_array(buffer) - } -} - -impl CompactEncoding for State { - fn preencode(&mut self, value: &Node) { - self.preencode(&value.index); - self.preencode(&value.length); - self.preencode_fixed_32(); - } - - fn encode(&mut self, value: &Node, buffer: &mut [u8]) { - self.encode(&value.index, buffer); - self.encode(&value.length, buffer); - self.encode_fixed_32(&value.hash, buffer); - } - - fn decode(&mut self, buffer: &[u8]) -> Node { - let index: u64 = self.decode(buffer); - let length: u64 = self.decode(buffer); - let hash: Box<[u8]> = self.decode_fixed_32(buffer); - Node::new(index, hash.to_vec(), length) - } -} - -impl CompactEncoding> for State { - fn preencode(&mut self, value: &Vec) { - let len = value.len(); - self.preencode(&len); - for val in value.into_iter() { - self.preencode(val); - } - } - - fn encode(&mut self, value: &Vec, buffer: &mut [u8]) { - let len = value.len(); - self.encode(&len, buffer); - for val in value { - self.encode(val, buffer); - } - } - - fn decode(&mut self, buffer: &[u8]) -> Vec { - let len: usize = self.decode(buffer); - let mut value = Vec::with_capacity(len); - for _ in 0..len { - value.push(self.decode(buffer)); - } - value - } -} +pub use types::{CompactEncoding, State}; diff --git a/src/compact_encoding/types.rs b/src/compact_encoding/types.rs new file mode 100644 index 00000000..cbd9134f --- /dev/null +++ b/src/compact_encoding/types.rs @@ -0,0 +1,376 @@ +//! Basic types of compact_encoding. +use std::convert::TryFrom; +use std::fmt::Debug; + +const U16_SIGNIFIER: u8 = 0xfd; +const U32_SIGNIFIER: u8 = 0xfe; +const U64_SIGNIFIER: u8 = 0xff; + +/// State. +#[derive(Debug, Clone)] +pub struct State { + /// Start position + pub start: usize, + /// End position + pub end: usize, +} + +impl State { + /// Create emtpy state + pub fn new() -> State { + State::new_with_start_and_end(0, 0) + } + + /// Create a state with an already known size. + /// With this, you can/must skip the preencode step. + pub fn new_with_size(size: usize) -> (State, Box<[u8]>) { + ( + State::new_with_start_and_end(0, size), + vec![0; size].into_boxed_slice(), + ) + } + + /// Create a state with a start and end already known. + pub fn new_with_start_and_end(start: usize, end: usize) -> State { + State { start, end } + } + + /// Create a state from existing buffer. + pub fn from_buffer(buffer: &[u8]) -> State { + State::new_with_start_and_end(0, buffer.len()) + } + + /// After calling preencode(), this allocates the right size buffer to the heap. + /// Follow this with the same number of encode() steps to fill the created buffer. + pub fn create_buffer(&self) -> Box<[u8]> { + vec![0; self.end].into_boxed_slice() + } + + /// Preencode a string slice + pub fn preencode_str(&mut self, value: &str) { + self.preencode_usize_var(&value.len()); + self.end += value.len(); + } + + /// Encode a string slice + pub fn encode_str(&mut self, value: &str, buffer: &mut [u8]) { + let len = value.len(); + self.encode_usize_var(&len, buffer); + buffer[self.start..self.start + len].copy_from_slice(value.as_bytes()); + self.start += len; + } + + /// Decode a String + pub fn decode_string(&mut self, buffer: &[u8]) -> String { + let len = self.decode_usize_var(buffer); + let value = std::str::from_utf8(&buffer[self.start..self.start + len]) + .expect("string is invalid UTF-8"); + self.start += len; + value.to_string() + } + + /// Preencode a variable length usigned int + pub fn preencode_uint_var + Ord>(&mut self, uint: &T) { + self.end += if *uint < T::from(U16_SIGNIFIER.into()) { + 1 + } else if *uint <= T::from(0xffff) { + 3 + } else if *uint <= T::from(0xffffffff) { + 5 + } else { + 9 + }; + } + + /// Decode a fixed length u8 + pub fn decode_u8(&mut self, buffer: &[u8]) -> u8 { + let value: u8 = buffer[self.start]; + self.start += 1; + value + } + + /// Decode a fixed length u16 + pub fn decode_u16(&mut self, buffer: &[u8]) -> u16 { + let value: u16 = + ((buffer[self.start] as u16) << 0) | ((buffer[self.start + 1] as u16) << 8); + self.start += 2; + value + } + + /// Encode a variable length u32 + pub fn encode_u32_var(&mut self, value: &u32, buffer: &mut [u8]) { + if *value < U16_SIGNIFIER.into() { + let bytes = value.to_le_bytes(); + buffer[self.start] = bytes[0]; + self.start += 1; + } else if *value <= 0xffff { + buffer[self.start] = U16_SIGNIFIER; + self.start += 1; + self.encode_uint16_bytes(&value.to_le_bytes(), buffer); + } else { + buffer[self.start] = U32_SIGNIFIER; + self.start += 1; + self.encode_u32(*value, buffer); + } + } + + /// Encode u32 to 4 LE bytes. + pub fn encode_u32(&mut self, uint: u32, buffer: &mut [u8]) { + self.encode_uint32_bytes(&uint.to_le_bytes(), buffer); + } + + /// Decode a variable length u32 + pub fn decode_u32_var(&mut self, buffer: &[u8]) -> u32 { + let first = buffer[self.start]; + self.start += 1; + if first < U16_SIGNIFIER { + first.into() + } else if first == U16_SIGNIFIER { + self.decode_u16(buffer).into() + } else { + self.decode_u32(buffer).into() + } + } + + /// Decode a fixed length u32 + pub fn decode_u32(&mut self, buffer: &[u8]) -> u32 { + let value: u32 = ((buffer[self.start] as u32) << 0) + | ((buffer[self.start + 1] as u32) << 8) + | ((buffer[self.start + 2] as u32) << 16) + | ((buffer[self.start + 3] as u32) << 24); + self.start += 4; + value + } + + /// Encode a variable length u64 + pub fn encode_u64_var(&mut self, value: &u64, buffer: &mut [u8]) { + if *value < U16_SIGNIFIER.into() { + let bytes = value.to_le_bytes(); + buffer[self.start] = bytes[0]; + self.start += 1; + } else if *value <= 0xffff { + buffer[self.start] = U16_SIGNIFIER; + self.start += 1; + self.encode_uint16_bytes(&value.to_le_bytes(), buffer); + } else if *value <= 0xffffffff { + buffer[self.start] = U32_SIGNIFIER; + self.start += 1; + self.encode_uint32_bytes(&value.to_le_bytes(), buffer); + } else { + buffer[self.start] = U64_SIGNIFIER; + self.start += 1; + self.encode_uint64_bytes(&value.to_le_bytes(), buffer); + } + } + + /// Encode u64 to 8 LE bytes. + pub fn encode_u64(&mut self, uint: u64, buffer: &mut [u8]) { + self.encode_uint64_bytes(&uint.to_le_bytes(), buffer); + } + + /// Decode a variable length u64 + pub fn decode_u64_var(&mut self, buffer: &[u8]) -> u64 { + let first = buffer[self.start]; + self.start += 1; + if first < U16_SIGNIFIER { + first.into() + } else if first == U16_SIGNIFIER { + self.decode_u16(buffer).into() + } else if first == U32_SIGNIFIER { + self.decode_u32(buffer).into() + } else { + self.decode_u64(buffer) + } + } + + /// Decode a fixed length u64 + pub fn decode_u64(&mut self, buffer: &[u8]) -> u64 { + let value: u64 = ((buffer[self.start] as u64) << 0) + | ((buffer[self.start + 1] as u64) << 8) + | ((buffer[self.start + 2] as u64) << 16) + | ((buffer[self.start + 3] as u64) << 24) + | ((buffer[self.start + 4] as u64) << 32) + | ((buffer[self.start + 5] as u64) << 40) + | ((buffer[self.start + 6] as u64) << 48) + | ((buffer[self.start + 7] as u64) << 56); + self.start += 8; + value + } + + /// Preencode a byte buffer + pub fn preencode_buffer(&mut self, value: &Box<[u8]>) { + let len = value.len(); + self.preencode_usize_var(&len); + self.end += len; + } + + /// Preencode a vector byte buffer + pub fn preencode_buffer_vec(&mut self, value: &Vec) { + let len = value.len(); + self.preencode_usize_var(&len); + self.end += len; + } + + /// Encode a byte buffer + pub fn encode_buffer(&mut self, value: &[u8], buffer: &mut [u8]) { + let len = value.len(); + self.encode_usize_var(&len, buffer); + buffer[self.start..self.start + len].copy_from_slice(value); + self.start += len; + } + + /// Decode a byte buffer + pub fn decode_buffer(&mut self, buffer: &[u8]) -> Box<[u8]> { + self.decode_buffer_vec(buffer).into_boxed_slice() + } + + /// Decode a vector byte buffer + pub fn decode_buffer_vec(&mut self, buffer: &[u8]) -> Vec { + let len = self.decode_usize_var(buffer); + let value = buffer[self.start..self.start + len].to_vec(); + self.start += value.len(); + value + } + + /// Preencode a fixed 32 byte buffer + pub fn preencode_fixed_32(&mut self) { + self.end += 32; + } + + /// Encode a fixed 32 byte buffer + pub fn encode_fixed_32(&mut self, value: &[u8], buffer: &mut [u8]) { + buffer[self.start..self.start + 32].copy_from_slice(value); + self.start += 32; + } + + /// Encode a fixed 32 byte buffer + pub fn decode_fixed_32(&mut self, buffer: &[u8]) -> Box<[u8]> { + let value = buffer[self.start..self.start + 32] + .to_vec() + .into_boxed_slice(); + self.start += 32; + value + } + + /// Preencode a string array + pub fn preencode_string_array(&mut self, value: &Vec) { + let len = value.len(); + self.preencode_usize_var(&len); + for string_value in value.into_iter() { + self.preencode_str(string_value); + } + } + + /// Encode a String array + pub fn encode_string_array(&mut self, value: &Vec, buffer: &mut [u8]) { + let len = value.len(); + self.encode_usize_var(&len, buffer); + for string_value in value { + self.encode_str(string_value, buffer); + } + } + + /// Decode a String array + pub fn decode_string_array(&mut self, buffer: &[u8]) -> Vec { + let len = self.decode_usize_var(buffer); + let mut value = Vec::with_capacity(len); + for _ in 0..len { + value.push(self.decode_string(buffer)); + } + value + } + + /// Encode a 2 byte unsigned integer + pub fn encode_uint16_bytes(&mut self, bytes: &[u8], buffer: &mut [u8]) { + buffer[self.start] = bytes[0]; + buffer[self.start + 1] = bytes[1]; + self.start += 2; + } + + /// Encode a 4 byte unsigned integer + pub fn encode_uint32_bytes(&mut self, bytes: &[u8], buffer: &mut [u8]) { + self.encode_uint16_bytes(bytes, buffer); + buffer[self.start] = bytes[2]; + buffer[self.start + 1] = bytes[3]; + self.start += 2; + } + + /// Encode an 8 byte unsigned integer + pub fn encode_uint64_bytes(&mut self, bytes: &[u8], buffer: &mut [u8]) { + self.encode_uint32_bytes(bytes, buffer); + buffer[self.start] = bytes[4]; + buffer[self.start + 1] = bytes[5]; + buffer[self.start + 2] = bytes[6]; + buffer[self.start + 3] = bytes[7]; + self.start += 4; + } + + /// Preencode a variable length usize + pub fn preencode_usize_var(&mut self, value: &usize) { + // TODO: This repeats the logic from above that works for u8 -> u64, but sadly not usize + self.end += if *value < U16_SIGNIFIER.into() { + 1 + } else if *value <= 0xffff { + 3 + } else if *value <= 0xffffffff { + 5 + } else { + 9 + }; + } + + /// Encode a variable length usize + pub fn encode_usize_var(&mut self, value: &usize, buffer: &mut [u8]) { + if *value <= 0xfc { + let bytes = value.to_le_bytes(); + buffer[self.start] = bytes[0]; + self.start += 1; + } else if *value <= 0xffff { + buffer[self.start] = U16_SIGNIFIER; + self.start += 1; + self.encode_uint16_bytes(&value.to_le_bytes(), buffer); + } else if *value <= 0xffffffff { + buffer[self.start] = U32_SIGNIFIER; + self.start += 1; + self.encode_uint32_bytes(&value.to_le_bytes(), buffer); + } else { + buffer[self.start] = U64_SIGNIFIER; + self.start += 1; + self.encode_uint64_bytes(&value.to_le_bytes(), buffer); + } + } + + /// Decode a variable length usize + pub fn decode_usize_var(&mut self, buffer: &[u8]) -> usize { + let first = buffer[self.start]; + self.start += 1; + // NB: the from_le_bytes needs a [u8; 2] and that can't be efficiently + // created from a byte slice. + if first < U16_SIGNIFIER { + first.into() + } else if first == U16_SIGNIFIER { + self.decode_u16(buffer).into() + } else if first == U32_SIGNIFIER { + usize::try_from(self.decode_u32(buffer)) + .expect("Attempted converting to a 32 bit usize on below 32 bit system") + } else { + usize::try_from(self.decode_u64(buffer)) + .expect("Attempted converting to a 64 bit usize on below 64 bit system") + } + } +} + +/// Compact Encoding +pub trait CompactEncoding +where + T: Debug, +{ + /// Preencode + fn preencode(&mut self, value: &T); + + /// Encode + fn encode(&mut self, value: &T, buffer: &mut [u8]); + + /// Decode + fn decode(&mut self, buffer: &[u8]) -> T; +} diff --git a/src/lib.rs b/src/lib.rs index 189bab77..f12fcf7c 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -72,7 +72,7 @@ mod tree; pub use crate::common::Node; #[cfg(feature = "v10")] -pub use crate::common::Store; +pub use crate::common::{DataBlock, DataHash, DataSeek, DataUpgrade, Store}; #[cfg(feature = "v10")] pub use crate::core::Hypercore; pub use crate::crypto::{generate_keypair, sign, verify, Signature}; From 0ff300a1bdc4bee7f955082e282ec7f5349d9ba4 Mon Sep 17 00:00:00 2001 From: Timo Tiuraniemi Date: Mon, 17 Oct 2022 12:37:08 +0300 Subject: [PATCH 070/157] Move peer-related requests also here, to make a method for generating a proof possible --- src/common/mod.rs | 6 ++-- src/common/{data.rs => peer.rs} | 29 +++++++++++++++-- src/compact_encoding/custom.rs | 55 ++++++++++++++++++++++++++++++++- src/lib.rs | 4 ++- 4 files changed, 88 insertions(+), 6 deletions(-) rename src/common/{data.rs => peer.rs} (59%) diff --git a/src/common/mod.rs b/src/common/mod.rs index 8a8f8cb4..b3d88108 100644 --- a/src/common/mod.rs +++ b/src/common/mod.rs @@ -1,7 +1,9 @@ -mod data; mod node; +mod peer; mod store; -pub use self::data::{DataBlock, DataHash, DataSeek, DataUpgrade}; pub use self::node::{Node, NodeByteRange}; +pub use self::peer::{ + DataBlock, DataHash, DataSeek, DataUpgrade, RequestBlock, RequestSeek, RequestUpgrade, +}; pub use self::store::{Store, StoreInfo, StoreInfoInstruction, StoreInfoType}; diff --git a/src/common/data.rs b/src/common/peer.rs similarity index 59% rename from src/common/data.rs rename to src/common/peer.rs index 2afda2fe..c711e760 100644 --- a/src/common/data.rs +++ b/src/common/peer.rs @@ -1,7 +1,32 @@ use crate::Node; #[derive(Debug, Clone, PartialEq)] -/// Block of data +/// Request of a DataBlock or DataHash from peer +pub struct RequestBlock { + /// Hypercore index + pub index: u64, + /// TODO: document + pub nodes: u64, +} + +#[derive(Debug, Clone, PartialEq)] +/// Request of a DataSeek from peer +pub struct RequestSeek { + /// TODO: document + pub bytes: u64, +} + +#[derive(Debug, Clone, PartialEq)] +/// Request of a DataUpgrade from peer +pub struct RequestUpgrade { + /// Hypercore start index + pub start: u64, + /// Length of elements + pub length: u64, +} + +#[derive(Debug, Clone, PartialEq)] +/// Block of data to peer pub struct DataBlock { /// Hypercore index pub index: u64, @@ -12,7 +37,7 @@ pub struct DataBlock { } #[derive(Debug, Clone, PartialEq)] -/// Data hash +/// Data hash to peer pub struct DataHash { /// Hypercore index pub index: u64, diff --git a/src/compact_encoding/custom.rs b/src/compact_encoding/custom.rs index 46fced17..b7ff0cc4 100644 --- a/src/compact_encoding/custom.rs +++ b/src/compact_encoding/custom.rs @@ -1,6 +1,8 @@ //! Hypercore-specific compact encodings use super::{CompactEncoding, State}; -use crate::{DataBlock, DataHash, DataSeek, DataUpgrade, Node}; +use crate::{ + DataBlock, DataHash, DataSeek, DataUpgrade, Node, RequestBlock, RequestSeek, RequestUpgrade, +}; impl CompactEncoding for State { fn preencode(&mut self, value: &Node) { @@ -50,6 +52,57 @@ impl CompactEncoding> for State { } } +impl CompactEncoding for State { + fn preencode(&mut self, value: &RequestBlock) { + self.preencode(&value.index); + self.preencode(&value.nodes); + } + + fn encode(&mut self, value: &RequestBlock, buffer: &mut [u8]) { + self.encode(&value.index, buffer); + self.encode(&value.nodes, buffer); + } + + fn decode(&mut self, buffer: &[u8]) -> RequestBlock { + let index: u64 = self.decode(buffer); + let nodes: u64 = self.decode(buffer); + RequestBlock { index, nodes } + } +} + +impl CompactEncoding for State { + fn preencode(&mut self, value: &RequestSeek) { + self.preencode(&value.bytes); + } + + fn encode(&mut self, value: &RequestSeek, buffer: &mut [u8]) { + self.encode(&value.bytes, buffer); + } + + fn decode(&mut self, buffer: &[u8]) -> RequestSeek { + let bytes: u64 = self.decode(buffer); + RequestSeek { bytes } + } +} + +impl CompactEncoding for State { + fn preencode(&mut self, value: &RequestUpgrade) { + self.preencode(&value.start); + self.preencode(&value.length); + } + + fn encode(&mut self, value: &RequestUpgrade, buffer: &mut [u8]) { + self.encode(&value.start, buffer); + self.encode(&value.length, buffer); + } + + fn decode(&mut self, buffer: &[u8]) -> RequestUpgrade { + let start: u64 = self.decode(buffer); + let length: u64 = self.decode(buffer); + RequestUpgrade { start, length } + } +} + impl CompactEncoding for State { fn preencode(&mut self, value: &DataBlock) { self.preencode(&value.index); diff --git a/src/lib.rs b/src/lib.rs index f12fcf7c..3b940030 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -72,7 +72,9 @@ mod tree; pub use crate::common::Node; #[cfg(feature = "v10")] -pub use crate::common::{DataBlock, DataHash, DataSeek, DataUpgrade, Store}; +pub use crate::common::{ + DataBlock, DataHash, DataSeek, DataUpgrade, RequestBlock, RequestSeek, RequestUpgrade, Store, +}; #[cfg(feature = "v10")] pub use crate::core::Hypercore; pub use crate::crypto::{generate_keypair, sign, verify, Signature}; From 3c2b3ccb5d015289b417f3d461954585c0fa85f6 Mon Sep 17 00:00:00 2001 From: Timo Tiuraniemi Date: Wed, 19 Oct 2022 11:07:37 +0300 Subject: [PATCH 071/157] Use flat_tree with missing methods implemented --- Cargo.toml | 2 +- src/tree/merkle_tree.rs | 17 +---------------- 2 files changed, 2 insertions(+), 17 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index a726ccfd..f86e51e5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -26,7 +26,7 @@ blake2-rfc = "0.2.18" byteorder = "1.3.4" ed25519-dalek = "1.0.1" anyhow = "1.0.26" -flat-tree = "5.0.0" +flat-tree = { git = "https://github.com/ttiurani/flat-tree", branch = "v10" } lazy_static = "1.4.0" memory-pager = "0.9.0" merkle-tree-stream = "0.12.0" diff --git a/src/tree/merkle_tree.rs b/src/tree/merkle_tree.rs index 7b0a9d35..8056788f 100644 --- a/src/tree/merkle_tree.rs +++ b/src/tree/merkle_tree.rs @@ -272,23 +272,8 @@ impl MerkleTree { let head = 2 * changeset.ancestors; let mut iter = flat_tree::Iterator::new(head - 2); loop { - // TODO: we should implement a contains() method in the Iterator - // similar to the Javascript - // https://github.com/mafintosh/flat-tree/blob/master/index.js#L152 - // then this would work: - // if iter.contains(head) && iter.index() < head { let index = iter.index(); - let factor = iter.factor(); - let contains: bool = if head > index { - head < (index + factor / 2) - } else { - if head < index { - head > (index - factor / 2) - } else { - true - } - }; - if contains && index < head { + if iter.contains(head) && index < head { self.unflushed.insert(index, Node::new_blank(index)); } From 3ba963d317b814ea25ca580f9ab06b9a7837dbf2 Mon Sep 17 00:00:00 2001 From: Timo Tiuraniemi Date: Wed, 19 Oct 2022 15:42:01 +0300 Subject: [PATCH 072/157] Somewhat working, but known buggy implementation of creating a proof --- src/common/mod.rs | 2 +- src/common/peer.rs | 18 + src/core.rs | 337 +++++++++++++++++- src/tree/merkle_tree.rs | 749 +++++++++++++++++++++++++++++++++++++++- tests/core.rs | 8 +- 5 files changed, 1107 insertions(+), 7 deletions(-) diff --git a/src/common/mod.rs b/src/common/mod.rs index b3d88108..8cba5590 100644 --- a/src/common/mod.rs +++ b/src/common/mod.rs @@ -4,6 +4,6 @@ mod store; pub use self::node::{Node, NodeByteRange}; pub use self::peer::{ - DataBlock, DataHash, DataSeek, DataUpgrade, RequestBlock, RequestSeek, RequestUpgrade, + DataBlock, DataHash, DataSeek, DataUpgrade, Proof, RequestBlock, RequestSeek, RequestUpgrade, }; pub use self::store::{Store, StoreInfo, StoreInfoInstruction, StoreInfoType}; diff --git a/src/common/peer.rs b/src/common/peer.rs index c711e760..de70dabe 100644 --- a/src/common/peer.rs +++ b/src/common/peer.rs @@ -1,3 +1,6 @@ +//! Types needed for passing information with with peers. +//! hypercore-protocol-rs uses these types and wraps them +//! into wire messages. use crate::Node; #[derive(Debug, Clone, PartialEq)] @@ -25,6 +28,21 @@ pub struct RequestUpgrade { pub length: u64, } +#[derive(Debug, Clone, PartialEq)] +/// Proof generated from corresponding requests +pub struct Proof { + /// Fork + pub fork: u64, + /// Data block + pub block: Option, + /// Data hash + pub hash: Option, + /// Data seek + pub seek: Option, + /// Data updrade + pub upgrade: Option, +} + #[derive(Debug, Clone, PartialEq)] /// Block of data to peer pub struct DataBlock { diff --git a/src/core.rs b/src/core.rs index 70045071..cd2120bc 100644 --- a/src/core.rs +++ b/src/core.rs @@ -2,12 +2,13 @@ use crate::{ bitfield_v10::Bitfield, - common::StoreInfo, + common::{Proof, StoreInfo}, crypto::generate_keypair, data::BlockStore, oplog::{Header, Oplog, MAX_OPLOG_ENTRIES_BYTE_SIZE}, storage_v10::{PartialKeypair, Storage}, tree::MerkleTree, + RequestBlock, RequestSeek, RequestUpgrade, }; use anyhow::{anyhow, Result}; use ed25519_dalek::Signature; @@ -378,6 +379,42 @@ where &self.key_pair } + /// Create a proof for given request + pub async fn create_proof( + &mut self, + block: Option, + hash: Option, + seek: Option, + upgrade: Option, + ) -> Result { + // TODO: Generalize Either response stack + let proof = match self.tree.create_proof( + block.as_ref(), + hash.as_ref(), + seek.as_ref(), + upgrade.as_ref(), + None, + )? { + Either::Right(value) => value, + Either::Left(instructions) => { + let infos = self.storage.read_infos(&instructions).await?; + match self.tree.create_proof( + block.as_ref(), + hash.as_ref(), + seek.as_ref(), + upgrade.as_ref(), + Some(&infos), + )? { + Either::Right(value) => value, + Either::Left(_) => { + return Err(anyhow!("Could not create proof")); + } + } + } + }; + Ok(proof) + } + fn should_flush_bitfield_and_tree_and_oplog(&mut self) -> bool { if self.skip_flush_count == 0 || self.oplog.entries_byte_length >= MAX_OPLOG_ENTRIES_BYTE_SIZE @@ -427,3 +464,301 @@ fn update_contiguous_length( header.contiguous_length = c; } } + +#[cfg(test)] +mod tests { + use super::*; + use random_access_memory::RandomAccessMemory; + + #[async_std::test] + async fn core_create_proof_block_only() -> Result<()> { + let mut hypercore = create_hypercore_with_data(10).await?; + + let proof = hypercore + .create_proof(Some(RequestBlock { index: 4, nodes: 2 }), None, None, None) + .await?; + let block = proof.block.unwrap(); + assert_eq!(proof.upgrade, None); + assert_eq!(proof.seek, None); + assert_eq!(block.index, 4); + assert_eq!(block.nodes.len(), 2); + assert_eq!(block.nodes[0].index, 10); + assert_eq!(block.nodes[1].index, 13); + Ok(()) + } + + #[async_std::test] + async fn core_create_proof_block_and_upgrade() -> Result<()> { + let mut hypercore = create_hypercore_with_data(10).await?; + let proof = hypercore + .create_proof( + Some(RequestBlock { index: 4, nodes: 0 }), + None, + None, + Some(RequestUpgrade { + start: 0, + length: 10, + }), + ) + .await?; + let block = proof.block.unwrap(); + let upgrade = proof.upgrade.unwrap(); + assert_eq!(proof.seek, None); + assert_eq!(block.index, 4); + assert_eq!(block.nodes.len(), 3); + assert_eq!(block.nodes[0].index, 10); + assert_eq!(block.nodes[1].index, 13); + assert_eq!(block.nodes[2].index, 3); + assert_eq!(upgrade.start, 0); + assert_eq!(upgrade.length, 10); + assert_eq!(upgrade.nodes.len(), 1); + assert_eq!(upgrade.nodes[0].index, 17); + assert_eq!(upgrade.additional_nodes.len(), 0); + Ok(()) + } + + #[async_std::test] + async fn core_create_proof_block_and_upgrade_and_additional() -> Result<()> { + let mut hypercore = create_hypercore_with_data(10).await?; + let proof = hypercore + .create_proof( + Some(RequestBlock { index: 4, nodes: 0 }), + None, + None, + Some(RequestUpgrade { + start: 0, + length: 8, + }), + ) + .await?; + let block = proof.block.unwrap(); + let upgrade = proof.upgrade.unwrap(); + assert_eq!(proof.seek, None); + assert_eq!(block.index, 4); + assert_eq!(block.nodes.len(), 3); + assert_eq!(block.nodes[0].index, 10); + assert_eq!(block.nodes[1].index, 13); + assert_eq!(block.nodes[2].index, 3); + assert_eq!(upgrade.start, 0); + assert_eq!(upgrade.length, 8); + assert_eq!(upgrade.nodes.len(), 0); + assert_eq!(upgrade.additional_nodes.len(), 1); + assert_eq!(upgrade.additional_nodes[0].index, 17); + Ok(()) + } + + #[async_std::test] + async fn core_create_proof_block_and_upgrade_from_existing_state() -> Result<()> { + let mut hypercore = create_hypercore_with_data(10).await?; + let proof = hypercore + .create_proof( + Some(RequestBlock { index: 1, nodes: 0 }), + None, + None, + Some(RequestUpgrade { + start: 1, + length: 9, + }), + ) + .await?; + let block = proof.block.unwrap(); + let upgrade = proof.upgrade.unwrap(); + assert_eq!(proof.seek, None); + assert_eq!(block.index, 1); + assert_eq!(block.nodes.len(), 0); + assert_eq!(upgrade.start, 1); + assert_eq!(upgrade.length, 9); + assert_eq!(upgrade.nodes.len(), 3); + assert_eq!(upgrade.nodes[0].index, 5); + assert_eq!(upgrade.nodes[1].index, 11); + assert_eq!(upgrade.nodes[2].index, 17); + assert_eq!(upgrade.additional_nodes.len(), 0); + Ok(()) + } + + #[async_std::test] + async fn core_create_proof_block_and_upgrade_from_existing_state_with_additional() -> Result<()> + { + let mut hypercore = create_hypercore_with_data(10).await?; + let proof = hypercore + .create_proof( + Some(RequestBlock { index: 1, nodes: 0 }), + None, + None, + Some(RequestUpgrade { + start: 1, + length: 5, + }), + ) + .await?; + let block = proof.block.unwrap(); + let upgrade = proof.upgrade.unwrap(); + assert_eq!(proof.seek, None); + assert_eq!(block.index, 1); + assert_eq!(block.nodes.len(), 0); + assert_eq!(upgrade.start, 1); + assert_eq!(upgrade.length, 5); + assert_eq!(upgrade.nodes.len(), 2); + assert_eq!(upgrade.nodes[0].index, 5); + assert_eq!(upgrade.nodes[1].index, 9); + assert_eq!(upgrade.additional_nodes.len(), 2); + assert_eq!(upgrade.additional_nodes[0].index, 13); + assert_eq!(upgrade.additional_nodes[1].index, 17); + Ok(()) + } + + #[async_std::test] + async fn core_create_proof_block_and_seek_1_no_upgrade() -> Result<()> { + let mut hypercore = create_hypercore_with_data(10).await?; + let proof = hypercore + .create_proof( + Some(RequestBlock { index: 4, nodes: 2 }), + None, + Some(RequestSeek { bytes: 8 }), + None, + ) + .await?; + let block = proof.block.unwrap(); + assert_eq!(proof.seek, None); // seek included in block + assert_eq!(proof.upgrade, None); + assert_eq!(block.index, 4); + assert_eq!(block.nodes.len(), 2); + assert_eq!(block.nodes[0].index, 10); + assert_eq!(block.nodes[1].index, 13); + Ok(()) + } + + #[async_std::test] + async fn core_create_proof_block_and_seek_2_no_upgrade() -> Result<()> { + let mut hypercore = create_hypercore_with_data(10).await?; + let proof = hypercore + .create_proof( + Some(RequestBlock { index: 4, nodes: 2 }), + None, + Some(RequestSeek { bytes: 10 }), + None, + ) + .await?; + let block = proof.block.unwrap(); + assert_eq!(proof.seek, None); // seek included in block + assert_eq!(proof.upgrade, None); + assert_eq!(block.index, 4); + assert_eq!(block.nodes.len(), 2); + assert_eq!(block.nodes[0].index, 10); + assert_eq!(block.nodes[1].index, 13); + Ok(()) + } + + #[async_std::test] + async fn core_create_proof_block_and_seek_3_no_upgrade() -> Result<()> { + let mut hypercore = create_hypercore_with_data(10).await?; + let proof = hypercore + .create_proof( + Some(RequestBlock { index: 4, nodes: 2 }), + None, + Some(RequestSeek { bytes: 13 }), + None, + ) + .await?; + let block = proof.block.unwrap(); + let seek = proof.seek.unwrap(); + assert_eq!(proof.upgrade, None); + assert_eq!(block.index, 4); + assert_eq!(block.nodes.len(), 1); + assert_eq!(block.nodes[0].index, 10); + assert_eq!(seek.nodes.len(), 2); + assert_eq!(seek.nodes[0].index, 12); + assert_eq!(seek.nodes[1].index, 14); + Ok(()) + } + + #[async_std::test] + async fn core_create_proof_block_and_seek_to_tree_no_upgrade() -> Result<()> { + let mut hypercore = create_hypercore_with_data(16).await?; + let proof = hypercore + .create_proof( + Some(RequestBlock { index: 0, nodes: 4 }), + None, + Some(RequestSeek { bytes: 26 }), + None, + ) + .await?; + let block = proof.block.unwrap(); + let seek = proof.seek.unwrap(); + assert_eq!(proof.upgrade, None); + assert_eq!(block.nodes.len(), 3); + assert_eq!(block.nodes[0].index, 2); + assert_eq!(block.nodes[1].index, 5); + assert_eq!(block.nodes[2].index, 11); + assert_eq!(seek.nodes.len(), 2); + assert_eq!(seek.nodes[0].index, 19); + assert_eq!(seek.nodes[1].index, 27); + Ok(()) + } + + #[async_std::test] + async fn core_create_proof_block_and_seek_with_upgrade() -> Result<()> { + let mut hypercore = create_hypercore_with_data(10).await?; + let proof = hypercore + .create_proof( + Some(RequestBlock { index: 4, nodes: 2 }), + None, + Some(RequestSeek { bytes: 13 }), + Some(RequestUpgrade { + start: 8, + length: 2, + }), + ) + .await?; + let block = proof.block.unwrap(); + let seek = proof.seek.unwrap(); + let upgrade = proof.upgrade.unwrap(); + assert_eq!(block.index, 4); + assert_eq!(block.nodes.len(), 1); + assert_eq!(block.nodes[0].index, 10); + assert_eq!(seek.nodes.len(), 2); + assert_eq!(seek.nodes[0].index, 12); + assert_eq!(seek.nodes[1].index, 14); + assert_eq!(upgrade.nodes.len(), 1); + assert_eq!(upgrade.nodes[0].index, 17); + assert_eq!(upgrade.additional_nodes.len(), 0); + Ok(()) + } + + #[async_std::test] + async fn core_create_proof_seek_with_upgrade() -> Result<()> { + let mut hypercore = create_hypercore_with_data(10).await?; + let proof = hypercore + .create_proof( + None, + None, + Some(RequestSeek { bytes: 13 }), + Some(RequestUpgrade { + start: 0, + length: 10, + }), + ) + .await?; + let seek = proof.seek.unwrap(); + let upgrade = proof.upgrade.unwrap(); + assert_eq!(proof.block, None); + assert_eq!(seek.nodes.len(), 4); + assert_eq!(seek.nodes[0].index, 12); + assert_eq!(seek.nodes[1].index, 14); + assert_eq!(seek.nodes[2].index, 9); + assert_eq!(seek.nodes[3].index, 3); + assert_eq!(upgrade.nodes.len(), 1); + assert_eq!(upgrade.nodes[0].index, 17); + assert_eq!(upgrade.additional_nodes.len(), 0); + Ok(()) + } + + async fn create_hypercore_with_data(length: u64) -> Result> { + let storage = Storage::new_memory().await?; + let mut hypercore = Hypercore::new(storage).await?; + for i in 0..length { + hypercore.append(format!("#{}", i).as_bytes()).await?; + } + Ok(hypercore) + } +} diff --git a/src/tree/merkle_tree.rs b/src/tree/merkle_tree.rs index 8056788f..4f03ea7a 100644 --- a/src/tree/merkle_tree.rs +++ b/src/tree/merkle_tree.rs @@ -4,13 +4,16 @@ use ed25519_dalek::Signature; use futures::future::Either; use crate::common::NodeByteRange; +use crate::common::Proof; use crate::compact_encoding::State; use crate::oplog::HeaderTree; -use crate::Store; use crate::{ common::{StoreInfo, StoreInfoInstruction}, Node, }; +use crate::{ + DataBlock, DataHash, DataSeek, DataUpgrade, RequestBlock, RequestSeek, RequestUpgrade, Store, +}; use super::MerkleTreeChangeset; @@ -257,6 +260,191 @@ impl MerkleTree { } } + /// Creates proof from a requests. + /// TODO: This is now just a clone of javascript's + /// https://github.com/hypercore-protocol/hypercore/blob/7e30a0fe353c70ada105840ec1ead6627ff521e7/lib/merkle-tree.js#L604 + /// The implementation should be rewritten to make it clearer. + pub fn create_proof( + &self, + block: Option<&RequestBlock>, + hash: Option<&RequestBlock>, + seek: Option<&RequestSeek>, + upgrade: Option<&RequestUpgrade>, + infos: Option<&[StoreInfo]>, + ) -> Result, Proof>> { + let nodes: Vec = infos_to_nodes(infos); + let mut instructions: Vec = Vec::new(); + let fork = self.fork; + let signature = self.signature; + let head = 2 * self.length; + let (from, to) = if let Some(upgrade) = upgrade.as_ref() { + let from = upgrade.start * 2; + (from, from + upgrade.length * 2) + } else { + (0, head) + }; + let indexed = normalize_indexed(block, hash); + + if from >= to || to > head { + return Err(anyhow!("Invalid upgrade")); + } + + let mut sub_tree = head; + let mut p = LocalProof { + indexed: None, + seek: None, + nodes: None, + upgrade: None, + additional_upgrade: None, + }; + let mut untrusted_sub_tree = false; + if let Some(indexed) = indexed.as_ref() { + if seek.is_some() && upgrade.is_some() && indexed.index >= from { + return Err(anyhow!( + "Cannot both do a seek and block/hash request when upgrading" + )); + } + + if let Some(upgrade) = upgrade.as_ref() { + untrusted_sub_tree = indexed.last_index < upgrade.start; + } else { + untrusted_sub_tree = true; + } + + if untrusted_sub_tree { + sub_tree = nodes_to_root(indexed.index, indexed.nodes, to)?; + let seek_root = if let Some(seek) = seek.as_ref() { + // TODO: This most likely now doesn't work correctly now, need to + // think about how to incrementally get nodes and keep track of what + // index has already been attempted to get from disk. + let index_or_instructions = + self.seek_untrusted_tree(sub_tree, seek.bytes, &nodes)?; + match index_or_instructions { + Either::Left(new_instructions) => { + instructions.extend(new_instructions); + return Ok(Either::Left(instructions.into_boxed_slice())); + } + Either::Right(index) => index, + } + } else { + head + }; + if let Either::Left(new_instructions) = self.block_and_seek_proof( + Some(indexed), + seek.is_some(), + seek_root, + sub_tree, + &mut p, + &nodes, + )? { + instructions.extend(new_instructions); + } + } else if upgrade.is_some() { + sub_tree = indexed.index; + } + } + if !untrusted_sub_tree { + if let Some(seek) = seek.as_ref() { + // TODO: This also most likely now doesn't work correctly + let index_or_instructions = self.seek_from_head(to, seek.bytes, &nodes)?; + sub_tree = match index_or_instructions { + Either::Left(new_instructions) => { + instructions.extend(new_instructions); + return Ok(Either::Left(instructions.into_boxed_slice())); + } + Either::Right(index) => index, + }; + } + } + + if upgrade.is_some() { + if let Either::Left(new_instructions) = self.upgrade_proof( + indexed.as_ref(), + seek.is_some(), + from, + to, + sub_tree, + &mut p, + &nodes, + )? { + instructions.extend(new_instructions); + } + + if head > to { + if let Either::Left(new_instructions) = + self.additional_upgrade_proof(to, head, &mut p, &nodes)? + { + instructions.extend(new_instructions); + } + } + } + + if instructions.is_empty() { + let (data_block, data_hash): (Option, Option) = + if let Some(block) = block.as_ref() { + // + ( + Some(DataBlock { + index: block.index, + value: vec![], // TODO: this needs to come in + nodes: p.nodes.unwrap(), // TODO: unwrap + }), + None, + ) + } else if let Some(hash) = hash.as_ref() { + // + ( + None, + Some(DataHash { + index: hash.index, + nodes: p.nodes.unwrap(), // TODO: unwrap + }), + ) + } else { + (None, None) + }; + + let data_seek: Option = if let Some(seek) = seek.as_ref() { + if let Some(p_seek) = p.seek { + Some(DataSeek { + bytes: seek.bytes, + nodes: p_seek, + }) + } else { + None + } + } else { + None + }; + + let data_upgrade: Option = if let Some(upgrade) = upgrade.as_ref() { + Some(DataUpgrade { + start: upgrade.start, + length: upgrade.length, + nodes: p.upgrade.unwrap(), // TODO: unwrap + additional_nodes: if let Some(additional_upgrade) = p.additional_upgrade { + additional_upgrade + } else { + vec![] + }, + signature: signature.unwrap().to_bytes().to_vec(), // TODO: unwrap + }) + } else { + None + }; + + Ok(Either::Right(Proof { + fork, + block: data_block, + hash: data_hash, + seek: data_seek, + upgrade: data_upgrade, + })) + } else { + Ok(Either::Left(instructions.into_boxed_slice())) + } + } + fn commitable(&self, changeset: &MerkleTreeChangeset) -> bool { let correct_length: bool = if changeset.upgraded { changeset.original_tree_length == self.length @@ -430,6 +618,513 @@ impl MerkleTree { 40, ))) } + + fn seek_from_head( + &self, + head: u64, + bytes: u64, + nodes: &Vec, + ) -> Result, u64>> { + let mut instructions: Vec = Vec::new(); + let mut roots = vec![]; + flat_tree::full_roots(head, &mut roots); + let mut bytes = bytes; + + for i in 0..roots.len() { + let root = roots[i]; + let node_or_instruction = self.get_node(root, nodes)?; + match node_or_instruction { + Either::Left(instruction) => { + instructions.push(instruction); + } + Either::Right(node) => { + if bytes == node.length { + return Ok(Either::Right(root)); + } + if bytes > node.length { + bytes -= node.length; + continue; + } + let instructions_or_result = self.seek_trusted_tree(root, bytes, nodes)?; + return match instructions_or_result { + Either::Left(new_instructions) => { + instructions.extend(new_instructions); + Ok(Either::Left(instructions)) + } + Either::Right(index) => Ok(Either::Right(index)), + }; + } + } + } + + if instructions.is_empty() { + Ok(Either::Right(head)) + } else { + Ok(Either::Left(instructions)) + } + } + + /// Trust that bytes are within the root tree and find the block at bytes. + fn seek_trusted_tree( + &self, + root: u64, + bytes: u64, + nodes: &Vec, + ) -> Result, u64>> { + if bytes == 0 { + return Ok(Either::Right(root)); + } + let mut iter = flat_tree::Iterator::new(root); + let mut instructions: Vec = Vec::new(); + let mut bytes = bytes; + while iter.index() & 1 != 0 { + let node_or_instruction = self.get_node(iter.left_child(), nodes)?; + match node_or_instruction { + Either::Left(instruction) => { + iter.parent(); + if nodes.is_empty() { + instructions.push(instruction); + } else { + // TODO: This is unfortunately not guaranteed to work because + // the nodes array might be filled in some cases with multiple steps + // from instructions. Should think of a better way to signal that + // this particular index has been attempted to be located, probably + // needs an incoming vector of attempted ids. + return Ok(Either::Right(iter.index())); + } + } + Either::Right(node) => { + if node.length == bytes { + return Ok(Either::Right(iter.index())); + } + if node.length > bytes { + continue; + } + bytes -= node.length; + iter.sibling(); + } + } + } + if instructions.is_empty() { + Ok(Either::Right(iter.index())) + } else { + Ok(Either::Left(instructions)) + } + } + + /// Try to find the block at bytes without trusting that it *is* within the root passed. + fn seek_untrusted_tree( + &self, + root: u64, + bytes: u64, + nodes: &Vec, + ) -> Result, u64>> { + let mut instructions: Vec = Vec::new(); + let offset_or_instructions = self.byte_offset_from_nodes(root, nodes)?; + let mut bytes = bytes; + match offset_or_instructions { + Either::Left(new_instructions) => { + instructions.extend(new_instructions); + } + Either::Right(offset) => { + if offset > bytes { + return Err(anyhow!("Invalid seek")); + } + if offset == bytes { + return Ok(Either::Right(root)); + } + bytes -= offset; + let node_or_instruction = self.get_node(root, nodes)?; + match node_or_instruction { + Either::Left(instruction) => { + instructions.push(instruction); + } + Either::Right(node) => { + if node.length <= bytes { + return Err(anyhow!("Invalid seek")); + } + } + } + } + } + let instructions_or_result = self.seek_trusted_tree(root, bytes, nodes)?; + match instructions_or_result { + Either::Left(new_instructions) => { + instructions.extend(new_instructions); + Ok(Either::Left(instructions)) + } + Either::Right(index) => Ok(Either::Right(index)), + } + } + + fn block_and_seek_proof( + &self, + indexed: Option<&NormalizedIndexed>, + is_seek: bool, + seek_root: u64, + root: u64, + p: &mut LocalProof, + nodes: &Vec, + ) -> Result, ()>> { + if let Some(indexed) = indexed { + let mut iter = flat_tree::Iterator::new(indexed.index); + let mut instructions: Vec = Vec::new(); + let mut p_nodes: Vec = Vec::new(); + + if !indexed.value { + let node_or_instruction = self.get_node(iter.index(), nodes)?; + match node_or_instruction { + Either::Left(instruction) => { + instructions.push(instruction); + } + Either::Right(node) => { + p_nodes.push(node); + } + } + } + while iter.index() != root { + iter.sibling(); + if is_seek && iter.contains(seek_root) && iter.index() != seek_root { + let success_or_instruction = + self.seek_proof(seek_root, iter.index(), p, nodes)?; + match success_or_instruction { + Either::Left(new_instructions) => { + instructions.extend(new_instructions); + } + _ => (), + } + } else { + let node_or_instruction = self.get_node(iter.index(), nodes)?; + match node_or_instruction { + Either::Left(instruction) => { + instructions.push(instruction); + } + Either::Right(node) => { + p_nodes.push(node); + } + } + } + + iter.parent(); + } + p.nodes = Some(p_nodes); + if instructions.is_empty() { + Ok(Either::Right(())) + } else { + Ok(Either::Left(instructions)) + } + } else { + self.seek_proof(seek_root, root, p, nodes) + } + } + + fn seek_proof( + &self, + seek_root: u64, + root: u64, + p: &mut LocalProof, + nodes: &Vec, + ) -> Result, ()>> { + let mut iter = flat_tree::Iterator::new(seek_root); + let mut instructions: Vec = Vec::new(); + let mut seek_nodes: Vec = Vec::new(); + let node_or_instruction = self.get_node(iter.index(), nodes)?; + match node_or_instruction { + Either::Left(instruction) => { + instructions.push(instruction); + } + Either::Right(node) => { + seek_nodes.push(node); + } + } + + while iter.index() != root { + iter.sibling(); + let node_or_instruction = self.get_node(iter.index(), nodes)?; + match node_or_instruction { + Either::Left(instruction) => { + instructions.push(instruction); + } + Either::Right(node) => { + seek_nodes.push(node); + } + } + iter.parent(); + } + p.seek = Some(seek_nodes); + if instructions.is_empty() { + Ok(Either::Right(())) + } else { + Ok(Either::Left(instructions)) + } + } + + fn upgrade_proof( + &self, + indexed: Option<&NormalizedIndexed>, + is_seek: bool, + from: u64, + to: u64, + sub_tree: u64, + p: &mut LocalProof, + nodes: &Vec, + ) -> Result, ()>> { + let mut instructions: Vec = Vec::new(); + let mut upgrade: Vec = Vec::new(); + let mut has_upgrade = false; + + if from == 0 { + has_upgrade = true; + } + + let mut iter = flat_tree::Iterator::new(0); + let mut has_full_root = iter.full_root(to); + while has_full_root { + // check if they already have the node + if iter.index() + iter.factor() / 2 < from { + iter.next_tree(); + has_full_root = iter.full_root(to); + continue; + } + + // connect existing tree + if !has_upgrade && iter.contains(from - 2) { + has_upgrade = true; + let root = iter.index(); + let target = from - 2; + + iter.seek(target); + + while iter.index() != root { + iter.sibling(); + if iter.index() > target { + if p.nodes.is_none() && p.seek.is_none() && iter.contains(sub_tree) { + let success_or_instructions = self.block_and_seek_proof( + indexed, + is_seek, + sub_tree, + iter.index(), + p, + nodes, + )?; + if let Either::Left(new_instructions) = success_or_instructions { + instructions.extend(new_instructions); + } + } else { + let node_or_instruction = self.get_node(iter.index(), nodes)?; + match node_or_instruction { + Either::Left(instruction) => { + instructions.push(instruction); + } + Either::Right(node) => upgrade.push(node), + } + } + } + iter.parent(); + } + + iter.next_tree(); + has_full_root = iter.full_root(to); + continue; + } + + if !has_upgrade { + has_upgrade = true; + } + + // if the subtree included is a child of this tree, include that one + // instead of a dup node + if p.nodes.is_none() && p.seek.is_none() && iter.contains(sub_tree) { + let success_or_instructions = + self.block_and_seek_proof(indexed, is_seek, sub_tree, iter.index(), p, nodes)?; + if let Either::Left(new_instructions) = success_or_instructions { + instructions.extend(new_instructions); + } + iter.next_tree(); + has_full_root = iter.full_root(to); + continue; + } + + // add root (can be optimised since the root might be in tree.roots) + let node_or_instruction = self.get_node(iter.index(), nodes)?; + match node_or_instruction { + Either::Left(instruction) => { + instructions.push(instruction); + } + Either::Right(node) => upgrade.push(node), + } + + iter.next_tree(); + has_full_root = iter.full_root(to); + } + + if has_upgrade { + p.upgrade = Some(upgrade); + } + + // if (from === 0) p.upgrade = [] + + // for (const ite = flat.iterator(0); ite.fullRoot(to); ite.nextTree()) { + // // check if they already have the node + // if (ite.index + ite.factor / 2 < from) continue + + // // connect existing tree + // if (p.upgrade === null && ite.contains(from - 2)) { + // p.upgrade = [] + + // const root = ite.index + // const target = from - 2 + + // ite.seek(target) + + // while (ite.index !== root) { + // ite.sibling() + // if (ite.index > target) { + // if (p.node === null && p.seek === null && ite.contains(subTree)) { + // blockAndSeekProof(tree, node, seek, subTree, ite.index, p) + // } else { + // p.upgrade.push(tree.get(ite.index)) + // } + // } + // ite.parent() + // } + + // continue + // } + + // if (p.upgrade === null) { + // p.upgrade = [] + // } + + // // if the subtree included is a child of this tree, include that one + // // instead of a dup node + // if (p.node === null && p.seek === null && ite.contains(subTree)) { + // blockAndSeekProof(tree, node, seek, subTree, ite.index, p) + // continue + // } + + // // add root (can be optimised since the root might be in tree.roots) + // p.upgrade.push(tree.get(ite.index)) + // } + if instructions.is_empty() { + Ok(Either::Right(())) + } else { + Ok(Either::Left(instructions)) + } + } + + fn additional_upgrade_proof( + &self, + from: u64, + to: u64, + p: &mut LocalProof, + nodes: &Vec, + ) -> Result, ()>> { + let mut instructions: Vec = Vec::new(); + let mut additional_upgrade: Vec = Vec::new(); + let mut has_additional_upgrade = false; + + if from == 0 { + has_additional_upgrade = true; + } + + let mut iter = flat_tree::Iterator::new(0); + let mut has_full_root = iter.full_root(to); + while has_full_root { + // check if they already have the node + if iter.index() + iter.factor() / 2 < from { + iter.next_tree(); + has_full_root = iter.full_root(to); + continue; + } + + // connect existing tree + if !has_additional_upgrade && iter.contains(from - 2) { + has_additional_upgrade = true; + let root = iter.index(); + let target = from - 2; + + iter.seek(target); + + while iter.index() != root { + iter.sibling(); + if iter.index() > target { + let node_or_instruction = self.get_node(iter.index(), nodes)?; + match node_or_instruction { + Either::Left(instruction) => { + instructions.push(instruction); + } + Either::Right(node) => additional_upgrade.push(node), + } + } + iter.parent(); + } + + iter.next_tree(); + has_full_root = iter.full_root(to); + continue; + } + + if !has_additional_upgrade { + has_additional_upgrade = true; + } + + // add root (can be optimised since the root is in tree.roots) + let node_or_instruction = self.get_node(iter.index(), nodes)?; + match node_or_instruction { + Either::Left(instruction) => { + instructions.push(instruction); + } + Either::Right(node) => additional_upgrade.push(node), + } + + iter.next_tree(); + has_full_root = iter.full_root(to); + } + + if has_additional_upgrade { + p.additional_upgrade = Some(additional_upgrade); + } + + // if (from === 0) p.additionalUpgrade = [] + + // for (const ite = flat.iterator(0); ite.fullRoot(to); ite.nextTree()) { + // // check if they already have the node + // if (ite.index + ite.factor / 2 < from) continue + + // // connect existing tree + // if (p.additionalUpgrade === null && ite.contains(from - 2)) { + // p.additionalUpgrade = [] + + // const root = ite.index + // const target = from - 2 + + // ite.seek(target) + + // while (ite.index !== root) { + // ite.sibling() + // if (ite.index > target) { + // p.additionalUpgrade.push(tree.get(ite.index)) + // } + // ite.parent() + // } + + // continue + // } + + // if (p.additionalUpgrade === null) { + // p.additionalUpgrade = [] + // } + + // // add root (can be optimised since the root is in tree.roots) + // p.additionalUpgrade.push(tree.get(ite.index)) + // } + if instructions.is_empty() { + Ok(Either::Right(())) + } else { + Ok(Either::Left(instructions)) + } + } } fn get_root_indices(header_tree_length: &u64) -> Vec { @@ -459,3 +1154,55 @@ fn infos_to_nodes(infos: Option<&[StoreInfo]>) -> Vec { None => vec![], } } + +#[derive(Debug, Copy, Clone)] +struct NormalizedIndexed { + pub value: bool, + pub index: u64, + pub nodes: u64, + pub last_index: u64, +} + +fn normalize_indexed( + block: Option<&RequestBlock>, + hash: Option<&RequestBlock>, +) -> Option { + if let Some(block) = block { + Some(NormalizedIndexed { + value: true, + index: block.index * 2, + nodes: block.nodes, + last_index: block.index, + }) + } else if let Some(hash) = hash { + Some(NormalizedIndexed { + value: false, + index: hash.index, + nodes: hash.nodes, + last_index: flat_tree::right_span(hash.index) / 2, + }) + } else { + None + } +} + +/// Struct to use for local building of proof +#[derive(Debug, Clone)] +struct LocalProof { + pub indexed: Option, + pub seek: Option>, + pub nodes: Option>, + pub upgrade: Option>, + pub additional_upgrade: Option>, +} + +fn nodes_to_root(index: u64, nodes: u64, head: u64) -> Result { + let mut iter = flat_tree::Iterator::new(index); + for _ in 0..nodes { + iter.parent(); + if iter.contains(head) { + return Err(anyhow!("Nodes is out of bounds")); + } + } + Ok(iter.index()) +} diff --git a/tests/core.rs b/tests/core.rs index 09e9bf36..b46c19ca 100644 --- a/tests/core.rs +++ b/tests/core.rs @@ -1,12 +1,13 @@ +#![cfg(feature = "v10")] + mod common; use anyhow::Result; use common::get_test_key_pair; -#[cfg(feature = "v10")] -use hypercore::{Hypercore, Storage}; +use hypercore::{Hypercore, RequestBlock, Storage}; +use random_access_memory::RandomAccessMemory; #[async_std::test] -#[cfg(feature = "v10")] async fn hypercore_new() -> Result<()> { let storage = Storage::new_memory().await?; let _hypercore = Hypercore::new(storage).await?; @@ -14,7 +15,6 @@ async fn hypercore_new() -> Result<()> { } #[async_std::test] -#[cfg(feature = "v10")] async fn hypercore_new_with_key_pair() -> Result<()> { let storage = Storage::new_memory().await?; let key_pair = get_test_key_pair(); From b06744afbfa4ea46dc7221602879f541e2d29ebf Mon Sep 17 00:00:00 2001 From: Timo Tiuraniemi Date: Wed, 19 Oct 2022 15:47:54 +0300 Subject: [PATCH 073/157] Remove forgotten JS debug --- src/tree/merkle_tree.rs | 77 ----------------------------------------- 1 file changed, 77 deletions(-) diff --git a/src/tree/merkle_tree.rs b/src/tree/merkle_tree.rs index 4f03ea7a..02166d42 100644 --- a/src/tree/merkle_tree.rs +++ b/src/tree/merkle_tree.rs @@ -962,50 +962,6 @@ impl MerkleTree { p.upgrade = Some(upgrade); } - // if (from === 0) p.upgrade = [] - - // for (const ite = flat.iterator(0); ite.fullRoot(to); ite.nextTree()) { - // // check if they already have the node - // if (ite.index + ite.factor / 2 < from) continue - - // // connect existing tree - // if (p.upgrade === null && ite.contains(from - 2)) { - // p.upgrade = [] - - // const root = ite.index - // const target = from - 2 - - // ite.seek(target) - - // while (ite.index !== root) { - // ite.sibling() - // if (ite.index > target) { - // if (p.node === null && p.seek === null && ite.contains(subTree)) { - // blockAndSeekProof(tree, node, seek, subTree, ite.index, p) - // } else { - // p.upgrade.push(tree.get(ite.index)) - // } - // } - // ite.parent() - // } - - // continue - // } - - // if (p.upgrade === null) { - // p.upgrade = [] - // } - - // // if the subtree included is a child of this tree, include that one - // // instead of a dup node - // if (p.node === null && p.seek === null && ite.contains(subTree)) { - // blockAndSeekProof(tree, node, seek, subTree, ite.index, p) - // continue - // } - - // // add root (can be optimised since the root might be in tree.roots) - // p.upgrade.push(tree.get(ite.index)) - // } if instructions.is_empty() { Ok(Either::Right(())) } else { @@ -1086,39 +1042,6 @@ impl MerkleTree { p.additional_upgrade = Some(additional_upgrade); } - // if (from === 0) p.additionalUpgrade = [] - - // for (const ite = flat.iterator(0); ite.fullRoot(to); ite.nextTree()) { - // // check if they already have the node - // if (ite.index + ite.factor / 2 < from) continue - - // // connect existing tree - // if (p.additionalUpgrade === null && ite.contains(from - 2)) { - // p.additionalUpgrade = [] - - // const root = ite.index - // const target = from - 2 - - // ite.seek(target) - - // while (ite.index !== root) { - // ite.sibling() - // if (ite.index > target) { - // p.additionalUpgrade.push(tree.get(ite.index)) - // } - // ite.parent() - // } - - // continue - // } - - // if (p.additionalUpgrade === null) { - // p.additionalUpgrade = [] - // } - - // // add root (can be optimised since the root is in tree.roots) - // p.additionalUpgrade.push(tree.get(ite.index)) - // } if instructions.is_empty() { Ok(Either::Right(())) } else { From cbffbc8ac211ba832b4765bd8cec1624b1f0aed7 Mon Sep 17 00:00:00 2001 From: Timo Tiuraniemi Date: Thu, 20 Oct 2022 11:14:48 +0300 Subject: [PATCH 074/157] Made room for allowing for reading nodes failing --- src/common/store.rs | 37 +++++++++-- src/storage_v10/mod.rs | 34 +++++++---- src/tree/merkle_tree.rs | 132 ++++++++++++++++++++++++++-------------- 3 files changed, 140 insertions(+), 63 deletions(-) diff --git a/src/common/store.rs b/src/common/store.rs index 34001fd2..8d3f332c 100644 --- a/src/common/store.rs +++ b/src/common/store.rs @@ -29,7 +29,9 @@ pub struct StoreInfo { pub(crate) index: u64, pub(crate) length: Option, pub(crate) data: Option>, - pub(crate) drop: bool, + /// When reading, indicates missing value (can be true only if allow_miss is given as instruction). + /// When writing indicates that the value should be dropped. + pub(crate) miss: bool, } impl StoreInfo { @@ -40,7 +42,18 @@ impl StoreInfo { index, length: Some(data.len() as u64), data: Some(data.into()), - drop: false, + miss: false, + } + } + + pub fn new_content_miss(store: Store, index: u64) -> Self { + Self { + store, + info_type: StoreInfoType::Content, + index, + length: None, + data: None, + miss: true, } } @@ -51,7 +64,7 @@ impl StoreInfo { index, length: Some(length), data: None, - drop: true, + miss: true, } } @@ -62,7 +75,7 @@ impl StoreInfo { index, length: None, data: None, - drop: true, + miss: true, } } @@ -73,7 +86,7 @@ impl StoreInfo { index, length: Some(length), data: None, - drop: false, + miss: false, } } } @@ -85,6 +98,7 @@ pub struct StoreInfoInstruction { pub(crate) info_type: StoreInfoType, pub(crate) index: u64, pub(crate) length: Option, + pub(crate) allow_miss: bool, } impl StoreInfoInstruction { @@ -94,6 +108,17 @@ impl StoreInfoInstruction { info_type: StoreInfoType::Content, index, length: Some(length), + allow_miss: false, + } + } + + pub fn new_content_allow_miss(store: Store, index: u64, length: u64) -> Self { + Self { + store, + info_type: StoreInfoType::Content, + index, + length: Some(length), + allow_miss: true, } } @@ -103,6 +128,7 @@ impl StoreInfoInstruction { info_type: StoreInfoType::Content, index: 0, length: None, + allow_miss: false, } } @@ -112,6 +138,7 @@ impl StoreInfoInstruction { info_type: StoreInfoType::Size, index, length: None, + allow_miss: false, } } } diff --git a/src/storage_v10/mod.rs b/src/storage_v10/mod.rs index 538e522b..99528ddd 100644 --- a/src/storage_v10/mod.rs +++ b/src/storage_v10/mod.rs @@ -115,15 +115,25 @@ where Some(length) => length, None => storage.len().await.map_err(|e| anyhow!(e))?, }; - let buf = storage - .read(instruction.index, length) - .await - .map_err(|e| anyhow!(e))?; - infos.push(StoreInfo::new_content( - instruction.store.clone(), - instruction.index, - &buf, - )); + let read_result = storage.read(instruction.index, length).await; + let info: StoreInfo = match read_result { + Ok(buf) => Ok(StoreInfo::new_content( + instruction.store.clone(), + instruction.index, + &buf, + )), + Err(e) => { + if instruction.allow_miss { + Ok(StoreInfo::new_content_miss( + instruction.store.clone(), + instruction.index, + )) + } else { + Err(anyhow!(e)) + } + } + }?; + infos.push(info); } StoreInfoType::Size => { let length = storage.len().await.map_err(|e| anyhow!(e))?; @@ -157,7 +167,7 @@ where } match info.info_type { StoreInfoType::Content => { - if !info.drop { + if !info.miss { if let Some(data) = &info.data { storage .write(info.index, &data.to_vec()) @@ -175,10 +185,10 @@ where } } StoreInfoType::Size => { - if info.drop { + if info.miss { storage.truncate(info.index).await.map_err(|e| anyhow!(e))?; } else { - panic!("Flushing a size that isn't drop, is not supported"); + panic!("Flushing a size that isn't miss, is not supported"); } } } diff --git a/src/tree/merkle_tree.rs b/src/tree/merkle_tree.rs index 02166d42..e091b2fa 100644 --- a/src/tree/merkle_tree.rs +++ b/src/tree/merkle_tree.rs @@ -2,6 +2,7 @@ use anyhow::Result; use anyhow::{anyhow, ensure}; use ed25519_dalek::Signature; use futures::future::Either; +use intmap::IntMap; use crate::common::NodeByteRange; use crate::common::Proof; @@ -26,7 +27,7 @@ pub struct MerkleTree { pub(crate) byte_length: u64, pub(crate) fork: u64, pub(crate) signature: Option, - unflushed: intmap::IntMap, + unflushed: IntMap, truncated: bool, truncate_to: u64, } @@ -87,7 +88,7 @@ impl MerkleTree { length, byte_length, fork: header_tree.fork, - unflushed: intmap::IntMap::new(), + unflushed: IntMap::new(), truncated: false, truncate_to: 0, signature: None, @@ -146,11 +147,11 @@ impl MerkleTree { ) -> Result, NodeByteRange>> { let index = self.validate_hypercore_index(hypercore_index)?; // Get nodes out of incoming infos - let nodes: Vec = infos_to_nodes(infos); + let nodes: IntMap> = infos_to_nodes(infos); // Start with getting the requested node, which will get the length // of the byte range - let length_result = self.get_node(index, &nodes)?; + let length_result = self.required_node(index, &nodes)?; // As for the offset, that might require fetching a lot more nodes whose // lengths to sum @@ -194,7 +195,7 @@ impl MerkleTree { ) -> Result, u64>> { let index = self.validate_hypercore_index(hypercore_index)?; // Get nodes out of incoming infos - let nodes: Vec = infos_to_nodes(infos); + let nodes: IntMap> = infos_to_nodes(infos); // Get offset let offset_result = self.byte_offset_from_nodes(index, &nodes)?; match offset_result { @@ -218,7 +219,7 @@ impl MerkleTree { let head = length * 2; let mut full_roots = vec![]; flat_tree::full_roots(head, &mut full_roots); - let nodes: Vec = infos_to_nodes(infos); + let nodes: IntMap> = infos_to_nodes(infos); let mut changeset = self.changeset(); let mut instructions: Vec = Vec::new(); @@ -231,7 +232,7 @@ impl MerkleTree { changeset.roots.pop(); } - let node_or_instruction = self.get_node(root, &nodes)?; + let node_or_instruction = self.required_node(root, &nodes)?; match node_or_instruction { Either::Left(instruction) => { instructions.push(instruction); @@ -272,7 +273,7 @@ impl MerkleTree { upgrade: Option<&RequestUpgrade>, infos: Option<&[StoreInfo]>, ) -> Result, Proof>> { - let nodes: Vec = infos_to_nodes(infos); + let nodes: IntMap> = infos_to_nodes(infos); let mut instructions: Vec = Vec::new(); let fork = self.fork; let signature = self.signature; @@ -541,7 +542,7 @@ impl MerkleTree { fn byte_offset_from_nodes( &self, index: u64, - nodes: &Vec, + nodes: &IntMap>, ) -> Result, u64>> { let index = if (index & 1) == 1 { flat_tree::left_span(index) @@ -566,7 +567,7 @@ impl MerkleTree { iter.left_child(); } else { let left_child = iter.left_child(); - let node_or_instruction = self.get_node(left_child, nodes)?; + let node_or_instruction = self.required_node(left_child, nodes)?; match node_or_instruction { Either::Left(instruction) => { instructions.push(instruction); @@ -590,40 +591,70 @@ impl MerkleTree { )) } - fn get_node( + fn required_node( &self, index: u64, - nodes: &Vec, + nodes: &IntMap>, ) -> Result> { + match self.node(index, nodes, false)? { + Either::Left(value) => Ok(Either::Left(value)), + Either::Right(node) => { + if let Some(node) = node { + Ok(Either::Right(node)) + } else { + Err(anyhow!("Node at {} was required", index)) + } + } + } + } + + fn node( + &self, + index: u64, + nodes: &IntMap>, + allow_miss: bool, + ) -> Result>> { // First check if unflushed already has the node if let Some(node) = self.unflushed.get(index) { if node.blank || (self.truncated && node.index >= 2 * self.truncate_to) { // The node is either blank or being deleted - return Err(anyhow!("Could not load node: {}", index)); + return if allow_miss { + Ok(Either::Right(None)) + } else { + Err(anyhow!("Could not load node: {}", index)) + }; } - return Ok(Either::Right(node.clone())); + return Ok(Either::Right(Some(node.clone()))); } // Then check if it's already in the incoming nodes - for node in nodes { - if node.index == index { - return Ok(Either::Right(node.clone())); + let result = nodes.get(index); + if let Some(node_maybe) = result { + if let Some(node) = node_maybe { + return Ok(Either::Right(Some(node.clone()))); + } else if allow_miss { + return Ok(Either::Right(None)); + } else { + return Err(anyhow!("Could not load node: {}", index)); } } // If not, retunr an instruction - Ok(Either::Left(StoreInfoInstruction::new_content( - Store::Tree, - 40 * index, - 40, - ))) + let offset = 40 * index; + let length = 40; + let info = if allow_miss { + StoreInfoInstruction::new_content_allow_miss(Store::Tree, offset, length) + } else { + StoreInfoInstruction::new_content(Store::Tree, offset, length) + }; + Ok(Either::Left(info)) } fn seek_from_head( &self, head: u64, bytes: u64, - nodes: &Vec, + nodes: &IntMap>, ) -> Result, u64>> { let mut instructions: Vec = Vec::new(); let mut roots = vec![]; @@ -632,7 +663,7 @@ impl MerkleTree { for i in 0..roots.len() { let root = roots[i]; - let node_or_instruction = self.get_node(root, nodes)?; + let node_or_instruction = self.required_node(root, nodes)?; match node_or_instruction { Either::Left(instruction) => { instructions.push(instruction); @@ -669,7 +700,7 @@ impl MerkleTree { &self, root: u64, bytes: u64, - nodes: &Vec, + nodes: &IntMap>, ) -> Result, u64>> { if bytes == 0 { return Ok(Either::Right(root)); @@ -678,7 +709,7 @@ impl MerkleTree { let mut instructions: Vec = Vec::new(); let mut bytes = bytes; while iter.index() & 1 != 0 { - let node_or_instruction = self.get_node(iter.left_child(), nodes)?; + let node_or_instruction = self.required_node(iter.left_child(), nodes)?; match node_or_instruction { Either::Left(instruction) => { iter.parent(); @@ -717,7 +748,7 @@ impl MerkleTree { &self, root: u64, bytes: u64, - nodes: &Vec, + nodes: &IntMap>, ) -> Result, u64>> { let mut instructions: Vec = Vec::new(); let offset_or_instructions = self.byte_offset_from_nodes(root, nodes)?; @@ -734,7 +765,7 @@ impl MerkleTree { return Ok(Either::Right(root)); } bytes -= offset; - let node_or_instruction = self.get_node(root, nodes)?; + let node_or_instruction = self.required_node(root, nodes)?; match node_or_instruction { Either::Left(instruction) => { instructions.push(instruction); @@ -764,7 +795,7 @@ impl MerkleTree { seek_root: u64, root: u64, p: &mut LocalProof, - nodes: &Vec, + nodes: &IntMap>, ) -> Result, ()>> { if let Some(indexed) = indexed { let mut iter = flat_tree::Iterator::new(indexed.index); @@ -772,7 +803,7 @@ impl MerkleTree { let mut p_nodes: Vec = Vec::new(); if !indexed.value { - let node_or_instruction = self.get_node(iter.index(), nodes)?; + let node_or_instruction = self.required_node(iter.index(), nodes)?; match node_or_instruction { Either::Left(instruction) => { instructions.push(instruction); @@ -794,7 +825,7 @@ impl MerkleTree { _ => (), } } else { - let node_or_instruction = self.get_node(iter.index(), nodes)?; + let node_or_instruction = self.required_node(iter.index(), nodes)?; match node_or_instruction { Either::Left(instruction) => { instructions.push(instruction); @@ -823,12 +854,12 @@ impl MerkleTree { seek_root: u64, root: u64, p: &mut LocalProof, - nodes: &Vec, + nodes: &IntMap>, ) -> Result, ()>> { let mut iter = flat_tree::Iterator::new(seek_root); let mut instructions: Vec = Vec::new(); let mut seek_nodes: Vec = Vec::new(); - let node_or_instruction = self.get_node(iter.index(), nodes)?; + let node_or_instruction = self.required_node(iter.index(), nodes)?; match node_or_instruction { Either::Left(instruction) => { instructions.push(instruction); @@ -840,7 +871,7 @@ impl MerkleTree { while iter.index() != root { iter.sibling(); - let node_or_instruction = self.get_node(iter.index(), nodes)?; + let node_or_instruction = self.required_node(iter.index(), nodes)?; match node_or_instruction { Either::Left(instruction) => { instructions.push(instruction); @@ -867,7 +898,7 @@ impl MerkleTree { to: u64, sub_tree: u64, p: &mut LocalProof, - nodes: &Vec, + nodes: &IntMap>, ) -> Result, ()>> { let mut instructions: Vec = Vec::new(); let mut upgrade: Vec = Vec::new(); @@ -911,7 +942,7 @@ impl MerkleTree { instructions.extend(new_instructions); } } else { - let node_or_instruction = self.get_node(iter.index(), nodes)?; + let node_or_instruction = self.required_node(iter.index(), nodes)?; match node_or_instruction { Either::Left(instruction) => { instructions.push(instruction); @@ -946,7 +977,7 @@ impl MerkleTree { } // add root (can be optimised since the root might be in tree.roots) - let node_or_instruction = self.get_node(iter.index(), nodes)?; + let node_or_instruction = self.required_node(iter.index(), nodes)?; match node_or_instruction { Either::Left(instruction) => { instructions.push(instruction); @@ -974,7 +1005,7 @@ impl MerkleTree { from: u64, to: u64, p: &mut LocalProof, - nodes: &Vec, + nodes: &IntMap>, ) -> Result, ()>> { let mut instructions: Vec = Vec::new(); let mut additional_upgrade: Vec = Vec::new(); @@ -1005,7 +1036,7 @@ impl MerkleTree { while iter.index() != root { iter.sibling(); if iter.index() > target { - let node_or_instruction = self.get_node(iter.index(), nodes)?; + let node_or_instruction = self.required_node(iter.index(), nodes)?; match node_or_instruction { Either::Left(instruction) => { instructions.push(instruction); @@ -1026,7 +1057,7 @@ impl MerkleTree { } // add root (can be optimised since the root is in tree.roots) - let node_or_instruction = self.get_node(iter.index(), nodes)?; + let node_or_instruction = self.required_node(iter.index(), nodes)?; match node_or_instruction { Either::Left(instruction) => { instructions.push(instruction); @@ -1068,13 +1099,22 @@ fn node_from_bytes(index: &u64, data: &[u8]) -> Node { Node::new(*index, hash.to_vec(), len) } -fn infos_to_nodes(infos: Option<&[StoreInfo]>) -> Vec { +fn infos_to_nodes(infos: Option<&[StoreInfo]>) -> IntMap> { match infos { - Some(infos) => infos - .iter() - .map(|info| node_from_bytes(&index_from_info(&info), info.data.as_ref().unwrap())) - .collect(), - None => vec![], + Some(infos) => { + let mut nodes: IntMap> = IntMap::with_capacity(infos.len()); + for info in infos { + let index = index_from_info(&info); + if !info.miss { + let node = node_from_bytes(&index, info.data.as_ref().unwrap()); + nodes.insert(index, Some(node)); + } else { + nodes.insert(index, None); + } + } + nodes + } + None => IntMap::new(), } } From 4ea2c40d6a3ffe9580ff845e0fb525d432bb919c Mon Sep 17 00:00:00 2001 From: Timo Tiuraniemi Date: Thu, 20 Oct 2022 11:46:17 +0300 Subject: [PATCH 075/157] Fix issue with not tolerating missing node --- src/core.rs | 36 +++++++++++++++++-------------- src/tree/merkle_tree.rs | 47 +++++++++++++++++++++++------------------ 2 files changed, 46 insertions(+), 37 deletions(-) diff --git a/src/core.rs b/src/core.rs index cd2120bc..c80cdac1 100644 --- a/src/core.rs +++ b/src/core.rs @@ -387,32 +387,36 @@ where seek: Option, upgrade: Option, ) -> Result { - // TODO: Generalize Either response stack - let proof = match self.tree.create_proof( + match self.tree.create_proof( block.as_ref(), hash.as_ref(), seek.as_ref(), upgrade.as_ref(), None, )? { - Either::Right(value) => value, + Either::Right(value) => return Ok(value), Either::Left(instructions) => { - let infos = self.storage.read_infos(&instructions).await?; - match self.tree.create_proof( - block.as_ref(), - hash.as_ref(), - seek.as_ref(), - upgrade.as_ref(), - Some(&infos), - )? { - Either::Right(value) => value, - Either::Left(_) => { - return Err(anyhow!("Could not create proof")); + let mut instructions = instructions; + let mut infos: Vec = vec![]; + loop { + infos.extend(self.storage.read_infos_to_vec(&instructions).await?); + match self.tree.create_proof( + block.as_ref(), + hash.as_ref(), + seek.as_ref(), + upgrade.as_ref(), + Some(&infos), + )? { + Either::Right(value) => { + return Ok(value); + } + Either::Left(new_instructions) => { + instructions = new_instructions; + } } } } - }; - Ok(proof) + } } fn should_flush_bitfield_and_tree_and_oplog(&mut self) -> bool { diff --git a/src/tree/merkle_tree.rs b/src/tree/merkle_tree.rs index e091b2fa..99cfcae0 100644 --- a/src/tree/merkle_tree.rs +++ b/src/tree/merkle_tree.rs @@ -315,9 +315,6 @@ impl MerkleTree { if untrusted_sub_tree { sub_tree = nodes_to_root(indexed.index, indexed.nodes, to)?; let seek_root = if let Some(seek) = seek.as_ref() { - // TODO: This most likely now doesn't work correctly now, need to - // think about how to incrementally get nodes and keep track of what - // index has already been attempted to get from disk. let index_or_instructions = self.seek_untrusted_tree(sub_tree, seek.bytes, &nodes)?; match index_or_instructions { @@ -608,6 +605,14 @@ impl MerkleTree { } } + fn optional_node( + &self, + index: u64, + nodes: &IntMap>, + ) -> Result>> { + self.node(index, nodes, true) + } + fn node( &self, index: u64, @@ -709,30 +714,30 @@ impl MerkleTree { let mut instructions: Vec = Vec::new(); let mut bytes = bytes; while iter.index() & 1 != 0 { - let node_or_instruction = self.required_node(iter.left_child(), nodes)?; + let node_or_instruction = self.optional_node(iter.left_child(), nodes)?; match node_or_instruction { Either::Left(instruction) => { - iter.parent(); - if nodes.is_empty() { - instructions.push(instruction); - } else { - // TODO: This is unfortunately not guaranteed to work because - // the nodes array might be filled in some cases with multiple steps - // from instructions. Should think of a better way to signal that - // this particular index has been attempted to be located, probably - // needs an incoming vector of attempted ids. - return Ok(Either::Right(iter.index())); - } + instructions.push(instruction); + // Need to break immediately because it is unknown + // if this node is the one that will match. This means + // this function needs to be called in a loop where incoming + // nodes increase with each call. + break; } Either::Right(node) => { - if node.length == bytes { + if let Some(node) = node { + if node.length == bytes { + return Ok(Either::Right(iter.index())); + } + if node.length > bytes { + continue; + } + bytes -= node.length; + iter.sibling(); + } else { + iter.parent(); return Ok(Either::Right(iter.index())); } - if node.length > bytes { - continue; - } - bytes -= node.length; - iter.sibling(); } } } From 0e07f054301903bdde2399cc4697e616abaae7d6 Mon Sep 17 00:00:00 2001 From: Timo Tiuraniemi Date: Thu, 20 Oct 2022 11:55:57 +0300 Subject: [PATCH 076/157] Proof creation should be working correctly now --- src/tree/merkle_tree.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/src/tree/merkle_tree.rs b/src/tree/merkle_tree.rs index 99cfcae0..47d69e1c 100644 --- a/src/tree/merkle_tree.rs +++ b/src/tree/merkle_tree.rs @@ -343,7 +343,6 @@ impl MerkleTree { } if !untrusted_sub_tree { if let Some(seek) = seek.as_ref() { - // TODO: This also most likely now doesn't work correctly let index_or_instructions = self.seek_from_head(to, seek.bytes, &nodes)?; sub_tree = match index_or_instructions { Either::Left(new_instructions) => { From 935e11df193c6f08749ba0f5c04311f62a9b4523 Mon Sep 17 00:00:00 2001 From: Timo Tiuraniemi Date: Fri, 21 Oct 2022 11:45:12 +0300 Subject: [PATCH 077/157] Untested implementation of verifying a proof received from a peer --- src/bitfield_v10/fixed.rs | 2 + src/core.rs | 42 ++++++- src/tree/merkle_tree.rs | 259 ++++++++++++++++++++++++++++++++++++++ 3 files changed, 302 insertions(+), 1 deletion(-) diff --git a/src/bitfield_v10/fixed.rs b/src/bitfield_v10/fixed.rs index fc827058..3492d080 100644 --- a/src/bitfield_v10/fixed.rs +++ b/src/bitfield_v10/fixed.rs @@ -10,6 +10,8 @@ use std::convert::TryInto; /// see: /// https://github.com/holepunchto/bits-to-bytes/blob/main/index.js /// for implementations. +/// TODO: This has been split into segments on the Javascript side "for improved disk performance": +/// https://github.com/hypercore-protocol/hypercore/commit/6392021b11d53041a446e9021c7d79350a052d3d #[derive(Debug)] pub struct FixedBitfield { pub(crate) parent_index: u64, diff --git a/src/core.rs b/src/core.rs index c80cdac1..8d91e85d 100644 --- a/src/core.rs +++ b/src/core.rs @@ -7,7 +7,7 @@ use crate::{ data::BlockStore, oplog::{Header, Oplog, MAX_OPLOG_ENTRIES_BYTE_SIZE}, storage_v10::{PartialKeypair, Storage}, - tree::MerkleTree, + tree::{MerkleTree, MerkleTreeChangeset}, RequestBlock, RequestSeek, RequestUpgrade, }; use anyhow::{anyhow, Result}; @@ -419,6 +419,23 @@ where } } + /// Verify a proof received from a peer. Returns a changeset that should be + /// applied. + async fn verify_proof(&mut self, proof: &Proof) -> Result { + match self.tree.verify_proof(proof, None)? { + Either::Right(value) => Ok(value), + Either::Left(instructions) => { + let infos = self.storage.read_infos_to_vec(&instructions).await?; + match self.tree.verify_proof(proof, Some(&infos))? { + Either::Right(value) => Ok(value), + Either::Left(_) => { + return Err(anyhow!("Could not read byte range")); + } + } + } + } + } + fn should_flush_bitfield_and_tree_and_oplog(&mut self) -> bool { if self.skip_flush_count == 0 || self.oplog.entries_byte_length >= MAX_OPLOG_ENTRIES_BYTE_SIZE @@ -757,6 +774,29 @@ mod tests { Ok(()) } + #[async_std::test] + async fn core_verify_proof_1() -> Result<()> { + let mut hypercore = create_hypercore_with_data(10).await?; + let mut hypercore_clone = create_hypercore_with_data(0).await?; + let proof = hypercore + .create_proof( + None, + Some(RequestBlock { index: 6, nodes: 0 }), + None, + Some(RequestUpgrade { + start: 0, + length: 10, + }), + ) + .await?; + let _changeset = hypercore_clone.verify_proof(&proof).await?; + + // TODO: Implement applying changeset and then test here that the end result matches + // in the clone. + + Ok(()) + } + async fn create_hypercore_with_data(length: u64) -> Result> { let storage = Storage::new_memory().await?; let mut hypercore = Hypercore::new(storage).await?; diff --git a/src/tree/merkle_tree.rs b/src/tree/merkle_tree.rs index 47d69e1c..cbb6dfbf 100644 --- a/src/tree/merkle_tree.rs +++ b/src/tree/merkle_tree.rs @@ -7,6 +7,7 @@ use intmap::IntMap; use crate::common::NodeByteRange; use crate::common::Proof; use crate::compact_encoding::State; +use crate::crypto::Hash; use crate::oplog::HeaderTree; use crate::{ common::{StoreInfo, StoreInfoInstruction}, @@ -442,6 +443,58 @@ impl MerkleTree { } } + /// Verifies a proof received from a peer. + pub fn verify_proof( + &self, + proof: &Proof, + infos: Option<&[StoreInfo]>, + ) -> Result, MerkleTreeChangeset>> { + let nodes: IntMap> = infos_to_nodes(infos); + let mut instructions: Vec = Vec::new(); + let mut changeset = self.changeset(); + + let mut unverified_block_root_node = verify_tree( + proof.block.as_ref(), + proof.hash.as_ref(), + proof.seek.as_ref(), + &mut changeset, + )?; + if let Some(upgrade) = proof.upgrade.as_ref() { + if verify_upgrade( + proof.fork, + upgrade, + unverified_block_root_node.as_ref(), + &mut changeset, + )? { + unverified_block_root_node = None; + } + } + + if let Some(unverified_block_root_node) = unverified_block_root_node { + let node_or_instruction = + self.required_node(unverified_block_root_node.index, &nodes)?; + match node_or_instruction { + Either::Left(instruction) => { + instructions.push(instruction); + } + Either::Right(verified_block_root_node) => { + if verified_block_root_node.hash != unverified_block_root_node.hash { + return Err(anyhow!( + "Invalid checksum at node {}", + unverified_block_root_node.index + )); + } + } + } + } + + if instructions.is_empty() { + Ok(Either::Right(changeset)) + } else { + Ok(Either::Left(instructions.into_boxed_slice())) + } + } + fn commitable(&self, changeset: &MerkleTreeChangeset) -> bool { let correct_length: bool = if changeset.upgraded { changeset.original_tree_length == self.length @@ -1085,6 +1138,132 @@ impl MerkleTree { } } +fn verify_tree( + block: Option<&DataBlock>, + hash: Option<&DataHash>, + seek: Option<&DataSeek>, + changeset: &mut MerkleTreeChangeset, +) -> Result> { + let untrusted_node: Option = normalize_data(block, hash); + + if untrusted_node.is_none() { + let no_seek = if let Some(seek) = seek.as_ref() { + seek.nodes.is_empty() + } else { + true + }; + if no_seek { + return Ok(None); + } + } + + let mut root: Option = None; + + if let Some(seek) = seek { + if !seek.nodes.is_empty() { + let mut iter = flat_tree::Iterator::new(seek.nodes[0].index); + let mut q = NodeQueue::new(seek.nodes.clone(), None); + let node = q.shift(iter.index())?; + let mut current_root: Node = node.clone(); + changeset.nodes.push(node); + while q.length > 0 { + let node = q.shift(iter.sibling())?; + let parent_node = parent_node(iter.parent(), ¤t_root, &node); + current_root = parent_node.clone(); + changeset.nodes.push(node); + changeset.nodes.push(parent_node); + } + root = Some(current_root); + } + } + + if let Some(untrusted_node) = untrusted_node { + let mut iter = flat_tree::Iterator::new(untrusted_node.index); + + let mut q = NodeQueue::new(untrusted_node.nodes, root); + let node: Node = if let Some(value) = untrusted_node.value { + block_node(iter.index(), &value) + } else { + q.shift(iter.index())? + }; + let mut current_root = node.clone(); + changeset.nodes.push(node); + while q.length > 0 { + let node = q.shift(iter.sibling())?; + let parent_node = parent_node(iter.parent(), ¤t_root, &node); + current_root = parent_node.clone(); + changeset.nodes.push(node); + changeset.nodes.push(parent_node); + } + root = Some(current_root); + } + Ok(root) +} + +fn verify_upgrade( + fork: u64, + upgrade: &DataUpgrade, + block_root: Option<&Node>, + changeset: &mut MerkleTreeChangeset, +) -> Result { + let mut q = if let Some(block_root) = block_root { + NodeQueue::new(upgrade.nodes.clone(), Some(block_root.clone())) + } else { + NodeQueue::new(upgrade.nodes.clone(), None) + }; + let mut grow: bool = changeset.roots.len() > 0; + let mut i: usize = 0; + let to: u64 = 2 * (upgrade.start + upgrade.length); + let mut iter = flat_tree::Iterator::new(0); + while iter.full_root(to) { + if i < changeset.roots.len() && changeset.roots[i].index == iter.index() { + i += 1; + iter.next_tree(); + continue; + } + if grow { + grow = false; + let root_index = iter.index(); + if i < changeset.roots.len() { + iter.seek(changeset.roots[changeset.roots.len() - 1].index); + while iter.index() != root_index { + changeset.append_root(q.shift(iter.sibling())?, &mut iter); + } + iter.next_tree(); + continue; + } + } + changeset.append_root(q.shift(iter.index())?, &mut iter); + iter.next_tree(); + } + let extra = &upgrade.additional_nodes; + + iter.seek(changeset.roots[changeset.roots.len() - 1].index); + i = 0; + + while i < extra.len() && extra[i].index == iter.sibling() { + changeset.append_root(extra[i].clone(), &mut iter); + i += 1; + } + + while i < extra.len() { + let node = extra[i].clone(); + i += 1; + while node.index != iter.index() { + if iter.factor() == 2 { + return Err(anyhow!("Unexpected node: {}", node.index)); + } + iter.left_child(); + } + changeset.append_root(node, &mut iter); + iter.sibling(); + } + changeset.signature = Some(Signature::from_bytes(&upgrade.signature)?); + changeset.fork = fork; + + Ok(q.extra.is_none()) +} + fn get_root_indices(header_tree_length: &u64) -> Vec { let mut roots = vec![]; flat_tree::full_roots(header_tree_length * 2, &mut roots); @@ -1153,6 +1332,31 @@ fn normalize_indexed( } } +#[derive(Debug, Clone)] +struct NormalizedData { + pub value: Option>, + pub index: u64, + pub nodes: Vec, +} + +fn normalize_data(block: Option<&DataBlock>, hash: Option<&DataHash>) -> Option { + if let Some(block) = block { + Some(NormalizedData { + value: Some(block.value.clone()), + index: block.index * 2, + nodes: block.nodes.clone(), + }) + } else if let Some(hash) = hash { + Some(NormalizedData { + value: None, + index: hash.index, + nodes: hash.nodes.clone(), + }) + } else { + None + } +} + /// Struct to use for local building of proof #[derive(Debug, Clone)] struct LocalProof { @@ -1173,3 +1377,58 @@ fn nodes_to_root(index: u64, nodes: u64, head: u64) -> Result { } Ok(iter.index()) } + +fn parent_node(index: u64, left: &Node, right: &Node) -> Node { + Node::new( + index, + Hash::parent(left, right).as_bytes().to_vec(), + left.length + right.length, + ) +} + +fn block_node(index: u64, value: &Vec) -> Node { + Node::new( + index, + Hash::data(value).as_bytes().to_vec(), + value.len() as u64, + ) +} + +/// Node queue +struct NodeQueue { + i: usize, + nodes: Vec, + extra: Option, + length: usize, +} +impl NodeQueue { + pub fn new(nodes: Vec, extra: Option) -> Self { + let length = nodes.len() + if extra.is_some() { 1 } else { 0 }; + Self { + i: 0, + nodes, + extra, + length, + } + } + pub fn shift(&mut self, index: u64) -> Result { + if let Some(extra) = self.extra.take() { + if extra.index == index { + self.length -= 1; + return Ok(extra); + } else { + self.extra = Some(extra); + } + } + if self.i >= self.nodes.len() { + return Err(anyhow!("Expected node {}, got (nil)", index)); + } + let node = self.nodes[self.i].clone(); + self.i += 1; + if node.index != index { + return Err(anyhow!("Expected node {}, got node {}", index, node.index)); + } + self.length -= 1; + Ok(node) + } +} From 54f4d9a298260db156a003378e234637615c3583 Mon Sep 17 00:00:00 2001 From: Timo Tiuraniemi Date: Fri, 21 Oct 2022 12:47:16 +0300 Subject: [PATCH 078/157] Verifying signature of a new proof works --- src/core.rs | 52 ++++++++++++++++++++++++++++--- src/tree/merkle_tree.rs | 3 +- src/tree/merkle_tree_changeset.rs | 7 ++++- 3 files changed, 56 insertions(+), 6 deletions(-) diff --git a/src/core.rs b/src/core.rs index 8d91e85d..34a3e72d 100644 --- a/src/core.rs +++ b/src/core.rs @@ -3,7 +3,7 @@ use crate::{ bitfield_v10::Bitfield, common::{Proof, StoreInfo}, - crypto::generate_keypair, + crypto::{generate_keypair, verify}, data::BlockStore, oplog::{Header, Oplog, MAX_OPLOG_ENTRIES_BYTE_SIZE}, storage_v10::{PartialKeypair, Storage}, @@ -419,6 +419,28 @@ where } } + /// Verify and apply proof received from peer, returns true if + pub async fn verify_and_apply_proof(&mut self, proof: &Proof) -> Result { + if proof.fork != self.tree.fork { + return Ok(false); + } + let changeset = self.verify_proof(proof).await?; + if !self.tree.commitable(&changeset) { + return Ok(false); + } + if changeset.upgraded { + // If this is committed, something will change, need to verify given + // new signature + verify( + &self.key_pair.public, + &changeset.signable(&changeset.hash()), + changeset.signature.as_ref(), + )?; + // TODO: + } + Ok(true) + } + /// Verify a proof received from a peer. Returns a changeset that should be /// applied. async fn verify_proof(&mut self, proof: &Proof) -> Result { @@ -777,7 +799,14 @@ mod tests { #[async_std::test] async fn core_verify_proof_1() -> Result<()> { let mut hypercore = create_hypercore_with_data(10).await?; - let mut hypercore_clone = create_hypercore_with_data(0).await?; + let mut hypercore_clone = create_hypercore_with_data_and_key_pair( + 0, + PartialKeypair { + public: hypercore.key_pair.public, + secret: None, + }, + ) + .await?; let proof = hypercore .create_proof( None, @@ -789,7 +818,7 @@ mod tests { }), ) .await?; - let _changeset = hypercore_clone.verify_proof(&proof).await?; + let _changeset = hypercore_clone.verify_and_apply_proof(&proof).await?; // TODO: Implement applying changeset and then test here that the end result matches // in the clone. @@ -798,8 +827,23 @@ mod tests { } async fn create_hypercore_with_data(length: u64) -> Result> { + let key_pair = generate_keypair(); + Ok(create_hypercore_with_data_and_key_pair( + length, + PartialKeypair { + public: key_pair.public, + secret: Some(key_pair.secret), + }, + ) + .await?) + } + + async fn create_hypercore_with_data_and_key_pair( + length: u64, + key_pair: PartialKeypair, + ) -> Result> { let storage = Storage::new_memory().await?; - let mut hypercore = Hypercore::new(storage).await?; + let mut hypercore = Hypercore::new_with_key_pair(storage, key_pair).await?; for i in 0..length { hypercore.append(format!("#{}", i).as_bytes()).await?; } diff --git a/src/tree/merkle_tree.rs b/src/tree/merkle_tree.rs index cbb6dfbf..dade85cd 100644 --- a/src/tree/merkle_tree.rs +++ b/src/tree/merkle_tree.rs @@ -495,7 +495,8 @@ impl MerkleTree { } } - fn commitable(&self, changeset: &MerkleTreeChangeset) -> bool { + /// Is the changeset commitable to given tree + pub fn commitable(&self, changeset: &MerkleTreeChangeset) -> bool { let correct_length: bool = if changeset.upgraded { changeset.original_tree_length == self.length } else { diff --git a/src/tree/merkle_tree_changeset.rs b/src/tree/merkle_tree_changeset.rs index 2c18fefb..f744ba82 100644 --- a/src/tree/merkle_tree_changeset.rs +++ b/src/tree/merkle_tree_changeset.rs @@ -87,7 +87,7 @@ impl MerkleTreeChangeset { /// Hashes and signs the changeset pub fn hash_and_sign(&mut self, public_key: &PublicKey, secret_key: &SecretKey) { let hash = self.hash(); - let signable = signable_tree(&hash, self.length, self.fork); + let signable = self.signable(&hash); let signature = sign(&public_key, &secret_key, &signable); self.hash = Some(hash); self.signature = Some(signature); @@ -97,4 +97,9 @@ impl MerkleTreeChangeset { pub fn hash(&self) -> Box<[u8]> { Hash::tree(&self.roots).as_bytes().into() } + + /// Creates a signable slice from given hash + pub fn signable(&self, hash: &[u8]) -> Box<[u8]> { + signable_tree(hash, self.length, self.fork) + } } From 1657373cd0b9be277c7d7e4b6a3694d159888d6b Mon Sep 17 00:00:00 2001 From: Timo Tiuraniemi Date: Mon, 24 Oct 2022 13:58:13 +0300 Subject: [PATCH 079/157] Verify and apply proof works, needs testing though --- src/bitfield_v10/dynamic.rs | 10 +- src/common/mod.rs | 8 + src/common/peer.rs | 35 +++- src/core.rs | 275 +++++++++++++++++++++--------- src/data/mod.rs | 4 + src/oplog/entry.rs | 33 +--- src/oplog/mod.rs | 69 ++++---- src/tree/merkle_tree.rs | 94 ++++++++-- src/tree/merkle_tree_changeset.rs | 19 ++- 9 files changed, 389 insertions(+), 158 deletions(-) diff --git a/src/bitfield_v10/dynamic.rs b/src/bitfield_v10/dynamic.rs index c3fbdc6b..394a80ed 100644 --- a/src/bitfield_v10/dynamic.rs +++ b/src/bitfield_v10/dynamic.rs @@ -1,6 +1,6 @@ use super::fixed::{FixedBitfield, FIXED_BITFIELD_BITS_LENGTH, FIXED_BITFIELD_LENGTH}; use crate::{ - common::{StoreInfo, StoreInfoInstruction, StoreInfoType}, + common::{BitfieldUpdate, StoreInfo, StoreInfoInstruction, StoreInfoType}, Store, }; use futures::future::Either; @@ -122,6 +122,14 @@ impl DynamicBitfield { changed } + pub fn update(&mut self, bitfield_update: &BitfieldUpdate) { + self.set_range( + bitfield_update.start, + bitfield_update.length, + !bitfield_update.drop, + ) + } + pub fn set_range(&mut self, start: u64, length: u64, value: bool) { let mut j = start & (DYNAMIC_BITFIELD_PAGE_SIZE as u64 - 1); let mut i = (start - j) / (DYNAMIC_BITFIELD_PAGE_SIZE as u64); diff --git a/src/common/mod.rs b/src/common/mod.rs index 8cba5590..d3b63c12 100644 --- a/src/common/mod.rs +++ b/src/common/mod.rs @@ -5,5 +5,13 @@ mod store; pub use self::node::{Node, NodeByteRange}; pub use self::peer::{ DataBlock, DataHash, DataSeek, DataUpgrade, Proof, RequestBlock, RequestSeek, RequestUpgrade, + ValuelessProof, }; pub use self::store::{Store, StoreInfo, StoreInfoInstruction, StoreInfoType}; + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct BitfieldUpdate { + pub(crate) drop: bool, + pub(crate) start: u64, + pub(crate) length: u64, +} diff --git a/src/common/peer.rs b/src/common/peer.rs index de70dabe..fb9bc333 100644 --- a/src/common/peer.rs +++ b/src/common/peer.rs @@ -33,7 +33,7 @@ pub struct RequestUpgrade { pub struct Proof { /// Fork pub fork: u64, - /// Data block + /// Data block. pub block: Option, /// Data hash pub hash: Option, @@ -43,6 +43,39 @@ pub struct Proof { pub upgrade: Option, } +#[derive(Debug, Clone, PartialEq)] +/// Valueless proof generated from corresponding requests +pub struct ValuelessProof { + /// Fork + pub fork: u64, + /// Data block. NB: The ValuelessProof struct uses the Hash type because + /// the stored binary value is processed externally to the proof. + pub block: Option, + /// Data hash + pub hash: Option, + /// Data seek + pub seek: Option, + /// Data updrade + pub upgrade: Option, +} + +impl ValuelessProof { + pub fn into_proof(&mut self, block_value: Option>) -> Proof { + let block = self.block.take().map(|block| DataBlock { + index: block.index, + nodes: block.nodes, + value: block_value.expect("Data block needs to be given"), + }); + Proof { + fork: self.fork, + block, + hash: self.hash.take(), + seek: self.seek.take(), + upgrade: self.upgrade.take(), + } + } +} + #[derive(Debug, Clone, PartialEq)] /// Block of data to peer pub struct DataBlock { diff --git a/src/core.rs b/src/core.rs index 34a3e72d..a5ecd466 100644 --- a/src/core.rs +++ b/src/core.rs @@ -2,8 +2,8 @@ use crate::{ bitfield_v10::Bitfield, - common::{Proof, StoreInfo}, - crypto::{generate_keypair, verify}, + common::{BitfieldUpdate, Proof, StoreInfo, ValuelessProof}, + crypto::generate_keypair, data::BlockStore, oplog::{Header, Oplog, MAX_OPLOG_ENTRIES_BYTE_SIZE}, storage_v10::{PartialKeypair, Storage}, @@ -132,18 +132,12 @@ where tree.add_node(node.clone()); } - if let Some(entry_bitfield) = &entry.bitfield { - bitfield.set_range( - entry_bitfield.start, - entry_bitfield.length, - !entry_bitfield.drop, - ); + if let Some(bitfield_update) = &entry.bitfield { + bitfield.update(&bitfield_update); update_contiguous_length( &mut oplog_open_outcome.header, &bitfield, - entry_bitfield.drop, - entry_bitfield.start, - entry_bitfield.length, + &bitfield_update, ); } if let Some(tree_upgrade) = &entry.tree_upgrade { @@ -227,25 +221,25 @@ where self.storage.flush_info(info).await?; // Append the changeset to the Oplog - let outcome = self.oplog.append_changeset(&changeset, false, &self.header); + let bitfield_update = BitfieldUpdate { + drop: false, + start: changeset.ancestors, + length: changeset.batch_length, + }; + let outcome = self.oplog.append_changeset( + &changeset, + Some(bitfield_update.clone()), + false, + &self.header, + ); self.storage.flush_infos(&outcome.infos_to_flush).await?; self.header = outcome.header; // Write to bitfield - self.bitfield.set_range( - changeset.ancestors, - changeset.length - changeset.ancestors, - true, - ); + self.bitfield.update(&bitfield_update); // Contiguous length is known only now - update_contiguous_length( - &mut self.header, - &self.bitfield, - false, - changeset.ancestors, - changeset.batch_length, - ); + update_contiguous_length(&mut self.header, &self.bitfield, &bitfield_update); // Commit changeset to in-memory tree self.tree.commit(changeset)?; @@ -386,8 +380,103 @@ where hash: Option, seek: Option, upgrade: Option, - ) -> Result { - match self.tree.create_proof( + ) -> Result> { + let mut valueless_proof = self + .create_valueless_proof(block, hash, seek, upgrade) + .await?; + let value: Option> = if let Some(block) = valueless_proof.block.as_ref() { + let value = self.get(block.index).await?; + if value.is_none() { + // The data value requested in the proof can not be read, we return None here + // and let the party requesting figure out what to do. + return Ok(None); + } + value + } else { + None + }; + Ok(Some(valueless_proof.into_proof(value))) + } + + /// Verify and apply proof received from peer, returns true if changed, false if not + /// possible to apply. + pub async fn verify_and_apply_proof(&mut self, proof: &Proof) -> Result { + if proof.fork != self.tree.fork { + return Ok(false); + } + let changeset = self.verify_proof(proof).await?; + if !self.tree.commitable(&changeset) { + return Ok(false); + } + + // In javascript there's _verifyExclusive and _verifyShared based on changeset.upgraded, but + // here we do only one. _verifyShared groups together many subsequent changesets into a single + // oplog push, and then flushes in the end only for the whole group. + let bitfield_update: Option = if let Some(block) = &proof.block.as_ref() { + let byte_offset = + match self + .tree + .byte_offset_in_changeset(block.index, &changeset, None)? + { + Either::Right(value) => value, + Either::Left(instructions) => { + let infos = self.storage.read_infos_to_vec(&instructions).await?; + match self.tree.byte_offset_in_changeset( + block.index, + &changeset, + Some(&infos), + )? { + Either::Right(value) => value, + Either::Left(_) => { + return Err(anyhow!("Could not read offset for index")); + } + } + } + }; + let info_to_flush = self.block_store.put(&block.value, byte_offset); + self.storage.flush_info(info_to_flush).await?; + Some(BitfieldUpdate { + drop: false, + start: block.index, + length: 1, + }) + } else { + None + }; + + // Append the changeset to the Oplog + let outcome = + self.oplog + .append_changeset(&changeset, bitfield_update.clone(), false, &self.header); + self.storage.flush_infos(&outcome.infos_to_flush).await?; + self.header = outcome.header; + + if let Some(bitfield_update) = bitfield_update { + // Write to bitfield + self.bitfield.update(&bitfield_update); + + // Contiguous length is known only now + update_contiguous_length(&mut self.header, &self.bitfield, &bitfield_update); + } + + // Commit changeset to in-memory tree + self.tree.commit(changeset)?; + + // Now ready to flush + if self.should_flush_bitfield_and_tree_and_oplog() { + self.flush_bitfield_and_tree_and_oplog().await?; + } + Ok(true) + } + + async fn create_valueless_proof( + &mut self, + block: Option, + hash: Option, + seek: Option, + upgrade: Option, + ) -> Result { + match self.tree.create_valueless_proof( block.as_ref(), hash.as_ref(), seek.as_ref(), @@ -400,7 +489,7 @@ where let mut infos: Vec = vec![]; loop { infos.extend(self.storage.read_infos_to_vec(&instructions).await?); - match self.tree.create_proof( + match self.tree.create_valueless_proof( block.as_ref(), hash.as_ref(), seek.as_ref(), @@ -419,39 +508,20 @@ where } } - /// Verify and apply proof received from peer, returns true if - pub async fn verify_and_apply_proof(&mut self, proof: &Proof) -> Result { - if proof.fork != self.tree.fork { - return Ok(false); - } - let changeset = self.verify_proof(proof).await?; - if !self.tree.commitable(&changeset) { - return Ok(false); - } - if changeset.upgraded { - // If this is committed, something will change, need to verify given - // new signature - verify( - &self.key_pair.public, - &changeset.signable(&changeset.hash()), - changeset.signature.as_ref(), - )?; - // TODO: - } - Ok(true) - } - /// Verify a proof received from a peer. Returns a changeset that should be /// applied. async fn verify_proof(&mut self, proof: &Proof) -> Result { - match self.tree.verify_proof(proof, None)? { + match self.tree.verify_proof(proof, &self.key_pair.public, None)? { Either::Right(value) => Ok(value), Either::Left(instructions) => { let infos = self.storage.read_infos_to_vec(&instructions).await?; - match self.tree.verify_proof(proof, Some(&infos))? { + match self + .tree + .verify_proof(proof, &self.key_pair.public, Some(&infos))? + { Either::Right(value) => Ok(value), Either::Left(_) => { - return Err(anyhow!("Could not read byte range")); + return Err(anyhow!("Could not verify key pair")); } } } @@ -484,18 +554,16 @@ where fn update_contiguous_length( header: &mut Header, bitfield: &Bitfield, - bitfield_drop: bool, - bitfield_start: u64, - bitfield_length: u64, + bitfield_update: &BitfieldUpdate, ) { - let end = bitfield_start + bitfield_length; + let end = bitfield_update.start + bitfield_update.length; let mut c = header.contiguous_length; - if bitfield_drop { - if c <= end && c > bitfield_start { - c = bitfield_start; + if bitfield_update.drop { + if c <= end && c > bitfield_update.start { + c = bitfield_update.start; } } else { - if c <= end && c >= bitfield_start { + if c <= end && c >= bitfield_update.start { c = end; while bitfield.get(c) { c += 1; @@ -519,7 +587,8 @@ mod tests { let proof = hypercore .create_proof(Some(RequestBlock { index: 4, nodes: 2 }), None, None, None) - .await?; + .await? + .unwrap(); let block = proof.block.unwrap(); assert_eq!(proof.upgrade, None); assert_eq!(proof.seek, None); @@ -543,7 +612,8 @@ mod tests { length: 10, }), ) - .await?; + .await? + .unwrap(); let block = proof.block.unwrap(); let upgrade = proof.upgrade.unwrap(); assert_eq!(proof.seek, None); @@ -573,7 +643,8 @@ mod tests { length: 8, }), ) - .await?; + .await? + .unwrap(); let block = proof.block.unwrap(); let upgrade = proof.upgrade.unwrap(); assert_eq!(proof.seek, None); @@ -603,7 +674,8 @@ mod tests { length: 9, }), ) - .await?; + .await? + .unwrap(); let block = proof.block.unwrap(); let upgrade = proof.upgrade.unwrap(); assert_eq!(proof.seek, None); @@ -633,7 +705,8 @@ mod tests { length: 5, }), ) - .await?; + .await? + .unwrap(); let block = proof.block.unwrap(); let upgrade = proof.upgrade.unwrap(); assert_eq!(proof.seek, None); @@ -660,7 +733,8 @@ mod tests { Some(RequestSeek { bytes: 8 }), None, ) - .await?; + .await? + .unwrap(); let block = proof.block.unwrap(); assert_eq!(proof.seek, None); // seek included in block assert_eq!(proof.upgrade, None); @@ -681,7 +755,8 @@ mod tests { Some(RequestSeek { bytes: 10 }), None, ) - .await?; + .await? + .unwrap(); let block = proof.block.unwrap(); assert_eq!(proof.seek, None); // seek included in block assert_eq!(proof.upgrade, None); @@ -702,7 +777,8 @@ mod tests { Some(RequestSeek { bytes: 13 }), None, ) - .await?; + .await? + .unwrap(); let block = proof.block.unwrap(); let seek = proof.seek.unwrap(); assert_eq!(proof.upgrade, None); @@ -725,7 +801,8 @@ mod tests { Some(RequestSeek { bytes: 26 }), None, ) - .await?; + .await? + .unwrap(); let block = proof.block.unwrap(); let seek = proof.seek.unwrap(); assert_eq!(proof.upgrade, None); @@ -752,7 +829,8 @@ mod tests { length: 2, }), ) - .await?; + .await? + .unwrap(); let block = proof.block.unwrap(); let seek = proof.seek.unwrap(); let upgrade = proof.upgrade.unwrap(); @@ -781,7 +859,8 @@ mod tests { length: 10, }), ) - .await?; + .await? + .unwrap(); let seek = proof.seek.unwrap(); let upgrade = proof.upgrade.unwrap(); assert_eq!(proof.block, None); @@ -797,17 +876,41 @@ mod tests { } #[async_std::test] - async fn core_verify_proof_1() -> Result<()> { + async fn core_verify_proof_invalid_signature() -> Result<()> { let mut hypercore = create_hypercore_with_data(10).await?; - let mut hypercore_clone = create_hypercore_with_data_and_key_pair( + // Invalid clone hypercore with a different public key + let mut hypercore_clone = create_hypercore_with_data(0).await?; + let proof = hypercore + .create_proof( + None, + Some(RequestBlock { index: 6, nodes: 0 }), + None, + Some(RequestUpgrade { + start: 0, + length: 10, + }), + ) + .await? + .unwrap(); + assert!(hypercore_clone + .verify_and_apply_proof(&proof) + .await + .is_err()); + Ok(()) + } + + #[async_std::test] + async fn core_verify_and_apply_proof() -> Result<()> { + let mut main = create_hypercore_with_data(10).await?; + let mut clone = create_hypercore_with_data_and_key_pair( 0, PartialKeypair { - public: hypercore.key_pair.public, + public: main.key_pair.public, secret: None, }, ) .await?; - let proof = hypercore + let proof = main .create_proof( None, Some(RequestBlock { index: 6, nodes: 0 }), @@ -817,12 +920,22 @@ mod tests { length: 10, }), ) - .await?; - let _changeset = hypercore_clone.verify_and_apply_proof(&proof).await?; - - // TODO: Implement applying changeset and then test here that the end result matches - // in the clone. - + .await? + .unwrap(); + assert!(clone.verify_and_apply_proof(&proof).await?); + let main_info = main.info(); + let clone_info = clone.info(); + assert_eq!(main_info.byte_length, clone_info.byte_length); + assert_eq!(main_info.length, clone_info.length); + assert!(main.get(6).await?.is_some()); + assert!(clone.get(6).await?.is_none()); + + // Fetch data for index 6 and verify it is found + let proof = main + .create_proof(Some(RequestBlock { index: 6, nodes: 2 }), None, None, None) + .await? + .unwrap(); + assert!(clone.verify_and_apply_proof(&proof).await?); Ok(()) } diff --git a/src/data/mod.rs b/src/data/mod.rs index c2c47a25..c8fb3889 100644 --- a/src/data/mod.rs +++ b/src/data/mod.rs @@ -19,6 +19,10 @@ impl BlockStore { StoreInfo::new_content(Store::Data, byte_length, &buffer) } + pub fn put(&self, value: &[u8], offset: u64) -> StoreInfo { + StoreInfo::new_content(Store::Data, offset, &value.to_vec()) + } + pub fn read( &self, byte_range: &NodeByteRange, diff --git a/src/oplog/entry.rs b/src/oplog/entry.rs index 05f1f815..eeb0ce1a 100644 --- a/src/oplog/entry.rs +++ b/src/oplog/entry.rs @@ -1,4 +1,5 @@ use crate::{ + common::BitfieldUpdate, compact_encoding::{CompactEncoding, State}, Node, }; @@ -41,22 +42,14 @@ impl CompactEncoding for State { } } -/// Entry bitfield update -#[derive(Debug)] -pub struct EntryBitfieldUpdate { - pub(crate) drop: bool, - pub(crate) start: u64, - pub(crate) length: u64, -} - -impl CompactEncoding for State { - fn preencode(&mut self, value: &EntryBitfieldUpdate) { +impl CompactEncoding for State { + fn preencode(&mut self, value: &BitfieldUpdate) { self.end += 1; self.preencode(&value.start); self.preencode(&value.length); } - fn encode(&mut self, value: &EntryBitfieldUpdate, buffer: &mut [u8]) { + fn encode(&mut self, value: &BitfieldUpdate, buffer: &mut [u8]) { let flags: u8 = if value.drop { 1 } else { 0 }; buffer[self.start] = flags; self.start += 1; @@ -64,11 +57,11 @@ impl CompactEncoding for State { self.encode(&value.length, buffer); } - fn decode(&mut self, buffer: &[u8]) -> EntryBitfieldUpdate { + fn decode(&mut self, buffer: &[u8]) -> BitfieldUpdate { let flags = self.decode_u8(buffer); let start: u64 = self.decode(buffer); let length: u64 = self.decode(buffer); - EntryBitfieldUpdate { + BitfieldUpdate { drop: flags == 1, start, length, @@ -79,19 +72,11 @@ impl CompactEncoding for State { /// Oplog Entry #[derive(Debug)] pub struct Entry { - // userData: null, - // treeNodes: batch.nodes, - // treeUpgrade: batch, - // bitfield: { - // drop: false, - // start: batch.ancestors, - // length: values.length - // } // TODO: This is a keyValueArray in JS pub(crate) user_data: Vec, pub(crate) tree_nodes: Vec, pub(crate) tree_upgrade: Option, - pub(crate) bitfield: Option, + pub(crate) bitfield: Option, } impl CompactEncoding for State { @@ -156,8 +141,8 @@ impl CompactEncoding for State { None }; - let bitfield: Option = if flags & 4 != 0 { - let value: EntryBitfieldUpdate = self.decode(buffer); + let bitfield: Option = if flags & 4 != 0 { + let value: BitfieldUpdate = self.decode(buffer); Some(value) } else { None diff --git a/src/oplog/mod.rs b/src/oplog/mod.rs index 33243f20..544f8857 100644 --- a/src/oplog/mod.rs +++ b/src/oplog/mod.rs @@ -1,4 +1,4 @@ -use crate::common::{Store, StoreInfo, StoreInfoInstruction}; +use crate::common::{BitfieldUpdate, Store, StoreInfo, StoreInfoInstruction}; use crate::compact_encoding::{CompactEncoding, State}; use crate::tree::MerkleTreeChangeset; use crate::{Node, PartialKeypair}; @@ -9,7 +9,7 @@ use std::convert::{TryFrom, TryInto}; mod entry; mod header; -pub use entry::{Entry, EntryBitfieldUpdate, EntryTreeUpgrade}; +pub use entry::{Entry, EntryTreeUpgrade}; pub use header::{Header, HeaderTree}; pub const MAX_OPLOG_ENTRIES_BYTE_SIZE: u64 = 65536; @@ -166,43 +166,48 @@ impl Oplog { } } - /// Appends a changeset to the Oplog. + /// Appends an upgraded changeset to the Oplog. pub fn append_changeset( &mut self, changeset: &MerkleTreeChangeset, + bitfield_update: Option, atomic: bool, header: &Header, ) -> OplogCreateHeaderOutcome { let tree_nodes: Vec = changeset.nodes.clone(); - let hash = changeset - .hash - .as_ref() - .expect("Changeset must have a hash before appended"); - let signature = changeset - .signature - .expect("Changeset must be signed before appended"); - let signature: Box<[u8]> = signature.to_bytes().into(); - - let entry: Entry = Entry { - user_data: vec![], - tree_nodes, - tree_upgrade: Some(EntryTreeUpgrade { - fork: changeset.fork, - ancestors: changeset.ancestors, - length: changeset.length, - signature: signature.clone(), - }), - bitfield: Some(EntryBitfieldUpdate { - drop: false, - start: changeset.ancestors, - length: changeset.batch_length, - }), - }; - let mut header: Header = header.clone(); - header.tree.length = changeset.length; - header.tree.root_hash = hash.clone(); - header.tree.signature = signature; + let entry: Entry = if changeset.upgraded { + let hash = changeset + .hash + .as_ref() + .expect("Upgraded changeset must have a hash before appended"); + let signature = changeset + .signature + .expect("Upgraded changeset must be signed before appended"); + let signature: Box<[u8]> = signature.to_bytes().into(); + header.tree.root_hash = hash.clone(); + header.tree.signature = signature.clone(); + header.tree.length = changeset.length; + + Entry { + user_data: vec![], + tree_nodes, + tree_upgrade: Some(EntryTreeUpgrade { + fork: changeset.fork, + ancestors: changeset.ancestors, + length: changeset.length, + signature, + }), + bitfield: bitfield_update, + } + } else { + Entry { + user_data: vec![], + tree_nodes, + tree_upgrade: None, + bitfield: bitfield_update, + } + }; OplogCreateHeaderOutcome { header, @@ -216,7 +221,7 @@ impl Oplog { user_data: vec![], tree_nodes: vec![], tree_upgrade: None, - bitfield: Some(EntryBitfieldUpdate { + bitfield: Some(BitfieldUpdate { drop: true, start, length: end - start, diff --git a/src/tree/merkle_tree.rs b/src/tree/merkle_tree.rs index dade85cd..984244b4 100644 --- a/src/tree/merkle_tree.rs +++ b/src/tree/merkle_tree.rs @@ -5,9 +5,9 @@ use futures::future::Either; use intmap::IntMap; use crate::common::NodeByteRange; -use crate::common::Proof; +use crate::common::{Proof, ValuelessProof}; use crate::compact_encoding::State; -use crate::crypto::Hash; +use crate::crypto::{Hash, PublicKey}; use crate::oplog::HeaderTree; use crate::{ common::{StoreInfo, StoreInfoInstruction}, @@ -207,6 +207,54 @@ impl MerkleTree { } } + /// Get the byte offset of hypercore index in a changeset + pub fn byte_offset_in_changeset( + &self, + hypercore_index: u64, + changeset: &MerkleTreeChangeset, + infos: Option<&[StoreInfo]>, + ) -> Result, u64>> { + if self.length == hypercore_index { + return Ok(Either::Right(self.byte_length)); + } + let mut iter = + flat_tree::Iterator::new(hypercore_index_into_merkle_tree_index(hypercore_index)); + let mut tree_offset = 0; + let mut is_right = false; + let mut parent: Option = None; + for node in &changeset.nodes { + if is_right { + if let Some(parent) = parent { + tree_offset += node.length - parent.length; + } + } + parent = Some(node.clone()); + is_right = iter.is_right(); + iter.parent(); + } + + let search_hypercore_index = if let Some(parent) = parent { + let r = changeset + .roots + .iter() + .position(|root| root.index == parent.index); + if let Some(r) = r { + for i in 0..r { + tree_offset += self.roots[i].length + } + return Ok(Either::Right(tree_offset)); + } + parent.index / 2 + } else { + hypercore_index + }; + + match self.byte_offset(search_hypercore_index, infos)? { + Either::Left(instructions) => Ok(Either::Left(instructions)), + Either::Right(offset) => Ok(Either::Right(offset + tree_offset)), + } + } + pub fn add_node(&mut self, node: Node) { self.unflushed.insert(node.index, node); } @@ -262,18 +310,18 @@ impl MerkleTree { } } - /// Creates proof from a requests. + /// Creates valueless proof from requests. /// TODO: This is now just a clone of javascript's /// https://github.com/hypercore-protocol/hypercore/blob/7e30a0fe353c70ada105840ec1ead6627ff521e7/lib/merkle-tree.js#L604 /// The implementation should be rewritten to make it clearer. - pub fn create_proof( + pub fn create_valueless_proof( &self, block: Option<&RequestBlock>, hash: Option<&RequestBlock>, seek: Option<&RequestSeek>, upgrade: Option<&RequestUpgrade>, infos: Option<&[StoreInfo]>, - ) -> Result, Proof>> { + ) -> Result, ValuelessProof>> { let nodes: IntMap> = infos_to_nodes(infos); let mut instructions: Vec = Vec::new(); let fork = self.fork; @@ -378,24 +426,21 @@ impl MerkleTree { } if instructions.is_empty() { - let (data_block, data_hash): (Option, Option) = + let (data_block, data_hash): (Option, Option) = if let Some(block) = block.as_ref() { - // ( - Some(DataBlock { + Some(DataHash { index: block.index, - value: vec![], // TODO: this needs to come in - nodes: p.nodes.unwrap(), // TODO: unwrap + nodes: p.nodes.expect("nodes need to be present"), }), None, ) } else if let Some(hash) = hash.as_ref() { - // ( None, Some(DataHash { index: hash.index, - nodes: p.nodes.unwrap(), // TODO: unwrap + nodes: p.nodes.expect("nodes need to be set"), }), ) } else { @@ -419,19 +464,22 @@ impl MerkleTree { Some(DataUpgrade { start: upgrade.start, length: upgrade.length, - nodes: p.upgrade.unwrap(), // TODO: unwrap + nodes: p.upgrade.expect("nodes need to be set"), additional_nodes: if let Some(additional_upgrade) = p.additional_upgrade { additional_upgrade } else { vec![] }, - signature: signature.unwrap().to_bytes().to_vec(), // TODO: unwrap + signature: signature + .expect("signature needs to be set") + .to_bytes() + .to_vec(), }) } else { None }; - Ok(Either::Right(Proof { + Ok(Either::Right(ValuelessProof { fork, block: data_block, hash: data_hash, @@ -447,6 +495,7 @@ impl MerkleTree { pub fn verify_proof( &self, proof: &Proof, + public_key: &PublicKey, infos: Option<&[StoreInfo]>, ) -> Result, MerkleTreeChangeset>> { let nodes: IntMap> = infos_to_nodes(infos); @@ -464,6 +513,7 @@ impl MerkleTree { proof.fork, upgrade, unverified_block_root_node.as_ref(), + public_key, &mut changeset, )? { unverified_block_root_node = None; @@ -571,7 +621,7 @@ impl MerkleTree { /// Validates given hypercore index and returns tree index fn validate_hypercore_index(&self, hypercore_index: u64) -> Result { // Converts a hypercore index into a merkle tree index - let index = 2 * hypercore_index; + let index = hypercore_index_into_merkle_tree_index(hypercore_index); // Check bounds let head = 2 * self.length; @@ -1139,6 +1189,14 @@ impl MerkleTree { } } +/// Converts a hypercore index into a merkle tree index. In the flat tree +/// representation, the leaves are in the even numbers, and the parents +/// odd. That's why we need to double the hypercore index value to get +/// the right merkle tree index. +fn hypercore_index_into_merkle_tree_index(hypercore_index: u64) -> u64 { + 2 * hypercore_index +} + fn verify_tree( block: Option<&DataBlock>, hash: Option<&DataHash>, @@ -1205,6 +1263,7 @@ fn verify_upgrade( fork: u64, upgrade: &DataUpgrade, block_root: Option<&Node>, + public_key: &PublicKey, changeset: &mut MerkleTreeChangeset, ) -> Result { let mut q = if let Some(block_root) = block_root { @@ -1259,9 +1318,8 @@ fn verify_upgrade( changeset.append_root(node, &mut iter); iter.sibling(); } - changeset.signature = Some(Signature::from_bytes(&upgrade.signature)?); changeset.fork = fork; - + changeset.verify_and_set_signature(&upgrade.signature, &public_key)?; Ok(q.extra.is_none()) } diff --git a/src/tree/merkle_tree_changeset.rs b/src/tree/merkle_tree_changeset.rs index f744ba82..58b33bae 100644 --- a/src/tree/merkle_tree_changeset.rs +++ b/src/tree/merkle_tree_changeset.rs @@ -1,7 +1,7 @@ use ed25519_dalek::{PublicKey, SecretKey, Signature}; use crate::{ - crypto::{signable_tree, Hash}, + crypto::{signable_tree, verify, Hash}, sign, Node, }; @@ -93,6 +93,23 @@ impl MerkleTreeChangeset { self.signature = Some(signature); } + /// Verify and set signature with given public key + pub fn verify_and_set_signature( + &mut self, + signature: &[u8], + public_key: &PublicKey, + ) -> anyhow::Result<()> { + // Verify that the received signature matches the public key + let signature = Signature::from_bytes(&signature)?; + let hash = self.hash(); + verify(&public_key, &self.signable(&hash), Some(&signature))?; + + // Set values to changeset + self.hash = Some(hash); + self.signature = Some(signature); + Ok(()) + } + /// Calculates a hash of the current set of roots pub fn hash(&self) -> Box<[u8]> { Hash::tree(&self.roots).as_bytes().into() From be257adec75f911143cd0636792c74a32ca4760a Mon Sep 17 00:00:00 2001 From: Timo Tiuraniemi Date: Mon, 24 Oct 2022 14:12:09 +0300 Subject: [PATCH 080/157] from_bytes didn't work, needs try_from --- src/core.rs | 5 +++++ src/tree/merkle_tree_changeset.rs | 3 ++- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/src/core.rs b/src/core.rs index a5ecd466..b95ef5fe 100644 --- a/src/core.rs +++ b/src/core.rs @@ -433,14 +433,19 @@ where } } }; + + // Write the value to the block store let info_to_flush = self.block_store.put(&block.value, byte_offset); self.storage.flush_info(info_to_flush).await?; + + // Return a bitfield update for the given value Some(BitfieldUpdate { drop: false, start: block.index, length: 1, }) } else { + // Only from DataBlock can there be changes to the bitfield None }; diff --git a/src/tree/merkle_tree_changeset.rs b/src/tree/merkle_tree_changeset.rs index 58b33bae..b72440e3 100644 --- a/src/tree/merkle_tree_changeset.rs +++ b/src/tree/merkle_tree_changeset.rs @@ -1,4 +1,5 @@ use ed25519_dalek::{PublicKey, SecretKey, Signature}; +use std::convert::TryFrom; use crate::{ crypto::{signable_tree, verify, Hash}, @@ -100,7 +101,7 @@ impl MerkleTreeChangeset { public_key: &PublicKey, ) -> anyhow::Result<()> { // Verify that the received signature matches the public key - let signature = Signature::from_bytes(&signature)?; + let signature = Signature::try_from(signature)?; let hash = self.hash(); verify(&public_key, &self.signable(&hash), Some(&signature))?; From 3d5927bcc85e7d9eba98b98ec3102ab1c082184e Mon Sep 17 00:00:00 2001 From: Timo Tiuraniemi Date: Mon, 24 Oct 2022 15:40:23 +0300 Subject: [PATCH 081/157] Fix storage.len() compilation failing upstream, fix issue with byte offset checking --- Cargo.toml | 2 +- src/lib.rs | 4 +++- src/tree/merkle_tree.rs | 16 +++++++++------- 3 files changed, 13 insertions(+), 9 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index f86e51e5..a3bfd502 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -33,7 +33,7 @@ merkle-tree-stream = "0.12.0" pretty-hash = "0.4.1" rand = "0.7.3" random-access-memory = { git = "https://github.com/ttiurani/random-access-memory", branch = "v10" } -random-access-storage = "4.0.0" +random-access-storage = { git = "https://github.com/ttiurani/random-access-storage" , branch = "v10"} sha2 = "0.9.2" sleep-parser = "0.8.0" sparse-bitfield = "0.11.0" diff --git a/src/lib.rs b/src/lib.rs index 3b940030..7b4889eb 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -73,7 +73,8 @@ mod tree; pub use crate::common::Node; #[cfg(feature = "v10")] pub use crate::common::{ - DataBlock, DataHash, DataSeek, DataUpgrade, RequestBlock, RequestSeek, RequestUpgrade, Store, + DataBlock, DataHash, DataSeek, DataUpgrade, Proof, RequestBlock, RequestSeek, RequestUpgrade, + Store, }; #[cfg(feature = "v10")] pub use crate::core::Hypercore; @@ -83,6 +84,7 @@ pub use crate::event::Event; pub use crate::feed::Feed; #[cfg(feature = "v9")] pub use crate::feed_builder::FeedBuilder; +#[cfg(feature = "v9")] pub use crate::proof::Proof; pub use crate::replicate::Peer; #[cfg(feature = "v9")] diff --git a/src/tree/merkle_tree.rs b/src/tree/merkle_tree.rs index 984244b4..e12cada3 100644 --- a/src/tree/merkle_tree.rs +++ b/src/tree/merkle_tree.rs @@ -223,14 +223,16 @@ impl MerkleTree { let mut is_right = false; let mut parent: Option = None; for node in &changeset.nodes { - if is_right { - if let Some(parent) = parent { - tree_offset += node.length - parent.length; + if node.index == iter.index() { + if is_right { + if let Some(parent) = parent { + tree_offset += node.length - parent.length; + } } + parent = Some(node.clone()); + is_right = iter.is_right(); + iter.parent(); } - parent = Some(node.clone()); - is_right = iter.is_right(); - iter.parent(); } let search_hypercore_index = if let Some(parent) = parent { @@ -242,7 +244,7 @@ impl MerkleTree { for i in 0..r { tree_offset += self.roots[i].length } - return Ok(Either::Right(tree_offset)); + return Ok(Either::Right(tree_offset as u64)); } parent.index / 2 } else { From d17ea017c8de235ef927b5b9ecef8f82b75cf628 Mon Sep 17 00:00:00 2001 From: Timo Tiuraniemi Date: Wed, 2 Nov 2022 10:48:20 +0200 Subject: [PATCH 082/157] Add compact encoding for Vec --- src/compact_encoding/generic.rs | 26 ++++++++++++++++++++++++++ src/tree/merkle_tree.rs | 7 ++++++- tests/compact_encoding.rs | 20 ++++++++++++++++++++ 3 files changed, 52 insertions(+), 1 deletion(-) diff --git a/src/compact_encoding/generic.rs b/src/compact_encoding/generic.rs index 809903c0..d0070af7 100644 --- a/src/compact_encoding/generic.rs +++ b/src/compact_encoding/generic.rs @@ -102,6 +102,32 @@ impl CompactEncoding> for State { } } +impl CompactEncoding> for State { + fn preencode(&mut self, value: &Vec) { + let len = value.len(); + self.preencode_usize_var(&len); + self.end += len * 4; + } + + fn encode(&mut self, value: &Vec, buffer: &mut [u8]) { + let len = value.len(); + self.encode_usize_var(&len, buffer); + for entry in value { + self.encode_u32(*entry, buffer); + } + self.start += len * 4; + } + + fn decode(&mut self, buffer: &[u8]) -> Vec { + let len = self.decode_usize_var(buffer); + let mut value: Vec = Vec::with_capacity(len); + for _ in 0..len { + value.push(self.decode_u32(&buffer)); + } + value + } +} + impl CompactEncoding> for State { fn preencode(&mut self, value: &Vec) { self.preencode_string_array(value); diff --git a/src/tree/merkle_tree.rs b/src/tree/merkle_tree.rs index e12cada3..d52b62b0 100644 --- a/src/tree/merkle_tree.rs +++ b/src/tree/merkle_tree.rs @@ -1433,7 +1433,12 @@ fn nodes_to_root(index: u64, nodes: u64, head: u64) -> Result { for _ in 0..nodes { iter.parent(); if iter.contains(head) { - return Err(anyhow!("Nodes is out of bounds")); + return Err(anyhow!( + "Nodes is out of bounds, index: {}, nodes: {}, head {}", + index, + nodes, + head + )); } } Ok(iter.index()) diff --git a/tests/compact_encoding.rs b/tests/compact_encoding.rs index 0d6042da..52d14dae 100644 --- a/tests/compact_encoding.rs +++ b/tests/compact_encoding.rs @@ -128,6 +128,26 @@ fn cenc_buffer() { assert_eq!(buf_value_2, buf_value_2_ret); } +#[test] +fn cenc_vec() { + let buf_value_1: Vec = vec![0xFF, 0x00]; + let buf_value_2: Vec = vec![0xFFFFFFFF, 0x11223344, 0x99887766]; + let mut enc_state = State::new(); + enc_state.preencode(&buf_value_1); + enc_state.preencode(&buf_value_2); + let mut buffer = enc_state.create_buffer(); + // 1 byte for length, 2 bytes for data + // 1 byte for length, 4*3 bytes for data + assert_eq!(buffer.len(), 1 + 2 + 1 + 12); + enc_state.encode(&buf_value_1, &mut buffer); + enc_state.encode(&buf_value_2, &mut buffer); + let mut dec_state = State::from_buffer(&buffer); + let buf_value_1_ret: Vec = dec_state.decode(&buffer); + let buf_value_2_ret: Vec = dec_state.decode(&buffer); + assert_eq!(buf_value_1, buf_value_1_ret); + assert_eq!(buf_value_2, buf_value_2_ret); +} + #[test] fn cenc_string_array() { let string_array_value = vec!["first".to_string(), "second".to_string()]; From 7ec84deef57f44a0c8f9748f45784fc315e211c4 Mon Sep 17 00:00:00 2001 From: Timo Tiuraniemi Date: Wed, 2 Nov 2022 11:19:10 +0200 Subject: [PATCH 083/157] Added cenc support for raw buffer --- src/compact_encoding/types.rs | 23 ++++++++++++++++++++++- tests/compact_encoding.rs | 20 ++++++++++++++++++++ 2 files changed, 42 insertions(+), 1 deletion(-) diff --git a/src/compact_encoding/types.rs b/src/compact_encoding/types.rs index cbd9134f..90e51a77 100644 --- a/src/compact_encoding/types.rs +++ b/src/compact_encoding/types.rs @@ -232,6 +232,27 @@ impl State { value } + /// Preencode a raw byte buffer. Only possible to use if this is the last value + /// of the State. + pub fn preencode_raw_buffer(&mut self, value: &Vec) { + self.end += value.len(); + } + + /// Encode a raw byte buffer. Only possible to use if this is the last value + /// of the State. + pub fn encode_raw_buffer(&mut self, value: &[u8], buffer: &mut [u8]) { + buffer[self.start..self.start + value.len()].copy_from_slice(value); + self.start += value.len(); + } + + /// Decode a raw byte buffer. Only possible to use if this is the last value + /// of the State. + pub fn decode_raw_buffer(&mut self, buffer: &[u8]) -> Vec { + let value = buffer[self.start..self.end].to_vec(); + self.start = self.end; + value + } + /// Preencode a fixed 32 byte buffer pub fn preencode_fixed_32(&mut self) { self.end += 32; @@ -243,7 +264,7 @@ impl State { self.start += 32; } - /// Encode a fixed 32 byte buffer + /// Decode a fixed 32 byte buffer pub fn decode_fixed_32(&mut self, buffer: &[u8]) -> Box<[u8]> { let value = buffer[self.start..self.start + 32] .to_vec() diff --git a/tests/compact_encoding.rs b/tests/compact_encoding.rs index 52d14dae..bcf10c15 100644 --- a/tests/compact_encoding.rs +++ b/tests/compact_encoding.rs @@ -163,3 +163,23 @@ fn cenc_string_array() { let string_array_value_ret: Vec = dec_state.decode(&buffer); assert_eq!(string_array_value, string_array_value_ret); } + +#[test] +fn cenc_fixed_and_raw() { + let buf_value_1: Vec = vec![0xFF; 32]; + let buf_value_2: Vec = vec![0xFF, 0x11, 0x99]; + let mut enc_state = State::new(); + enc_state.preencode_fixed_32(); + enc_state.preencode_raw_buffer(&buf_value_2); + let mut buffer = enc_state.create_buffer(); + // 32 bytes for data + // 3 bytes for data + assert_eq!(buffer.len(), 32 + 3); + enc_state.encode_fixed_32(&buf_value_1, &mut buffer); + enc_state.encode_raw_buffer(&buf_value_2, &mut buffer); + let mut dec_state = State::from_buffer(&buffer); + let buf_value_1_ret: Vec = dec_state.decode_fixed_32(&buffer).to_vec(); + let buf_value_2_ret: Vec = dec_state.decode_raw_buffer(&buffer); + assert_eq!(buf_value_1, buf_value_1_ret); + assert_eq!(buf_value_2, buf_value_2_ret); +} From 4e801252b800288e5d7db506232bbfc938023d6b Mon Sep 17 00:00:00 2001 From: Timo Tiuraniemi Date: Wed, 2 Nov 2022 13:56:35 +0200 Subject: [PATCH 084/157] Fix Vec double start increase --- src/compact_encoding/generic.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/src/compact_encoding/generic.rs b/src/compact_encoding/generic.rs index d0070af7..78fdea67 100644 --- a/src/compact_encoding/generic.rs +++ b/src/compact_encoding/generic.rs @@ -115,7 +115,6 @@ impl CompactEncoding> for State { for entry in value { self.encode_u32(*entry, buffer); } - self.start += len * 4; } fn decode(&mut self, buffer: &[u8]) -> Vec { From 7e5783bc98cf33ea316b1ef6278f3d3ae2b63786 Mon Sep 17 00:00:00 2001 From: Timo Tiuraniemi Date: Thu, 3 Nov 2022 11:19:24 +0200 Subject: [PATCH 085/157] Fix v9 compilation and tests --- src/crypto/hash.rs | 8 ++++++++ src/crypto/mod.rs | 4 +++- src/lib.rs | 1 + tests/compact_encoding.rs | 2 ++ 4 files changed, 14 insertions(+), 1 deletion(-) diff --git a/src/crypto/hash.rs b/src/crypto/hash.rs index c33417e9..4688a689 100644 --- a/src/crypto/hash.rs +++ b/src/crypto/hash.rs @@ -1,6 +1,7 @@ pub use blake2_rfc::blake2b::Blake2bResult; use crate::common::Node; +#[cfg(feature = "v10")] use crate::compact_encoding::State; use blake2_rfc::blake2b::Blake2b; use byteorder::{BigEndian, WriteBytesExt}; @@ -103,6 +104,7 @@ impl Hash { // for v10 that use LE bytes. /// Hash data + #[cfg(feature = "v10")] pub fn data(data: &[u8]) -> Self { let (mut state, mut size) = State::new_with_size(8); state.encode_u64(data.len() as u64, &mut size); @@ -118,6 +120,7 @@ impl Hash { } /// Hash a parent + #[cfg(feature = "v10")] pub fn parent(left: &Node, right: &Node) -> Self { let (node1, node2) = if left.index <= right.index { (left, right) @@ -140,6 +143,7 @@ impl Hash { } /// Hash a tree + #[cfg(feature = "v10")] pub fn tree(roots: &[impl AsRef]) -> Self { let mut hasher = Blake2b::new(32); hasher.update(&ROOT_TYPE); @@ -183,6 +187,7 @@ impl DerefMut for Hash { /// Create a signable buffer for tree. This is treeSignable in Javascript. /// See https://github.com/hypercore-protocol/hypercore/blob/70b271643c4e4b1e5ecae5bb579966dfe6361ff3/lib/caps.js#L17 +#[cfg(feature = "v10")] pub fn signable_tree(hash: &[u8], length: u64, fork: u64) -> Box<[u8]> { let (mut state, mut buffer) = State::new_with_size(80); state.encode_fixed_32(&TREE, &mut buffer); @@ -280,6 +285,7 @@ mod tests { // https://github.com/mafintosh/hypercore-crypto/blob/master/test.js #[test] + #[cfg(feature = "v10")] fn hash_leaf() { let data = b"hello world"; check_hash( @@ -289,6 +295,7 @@ mod tests { } #[test] + #[cfg(feature = "v10")] fn hash_parent() { let data = b"hello world"; let len = data.len() as u64; @@ -301,6 +308,7 @@ mod tests { } #[test] + #[cfg(feature = "v10")] fn hash_tree() { let hash: [u8; 32] = [0; 32]; let node1 = Node::new(3, hash.to_vec(), 11); diff --git a/src/crypto/mod.rs b/src/crypto/mod.rs index abfe4397..4cc10858 100644 --- a/src/crypto/mod.rs +++ b/src/crypto/mod.rs @@ -4,7 +4,9 @@ mod hash; mod key_pair; mod merkle; -pub use self::hash::{signable_tree, Hash}; +#[cfg(feature = "v10")] +pub use self::hash::signable_tree; +pub use self::hash::Hash; pub use self::key_pair::{ generate as generate_keypair, sign, verify, PublicKey, SecretKey, Signature, }; diff --git a/src/lib.rs b/src/lib.rs index 7b4889eb..a7650bd2 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -41,6 +41,7 @@ #[cfg(feature = "v9")] pub mod bitfield; +#[cfg(feature = "v10")] pub mod compact_encoding; pub mod prelude; diff --git a/tests/compact_encoding.rs b/tests/compact_encoding.rs index bcf10c15..7df59029 100644 --- a/tests/compact_encoding.rs +++ b/tests/compact_encoding.rs @@ -1,3 +1,5 @@ +#![cfg(feature = "v10")] + use hypercore::compact_encoding::{CompactEncoding, State}; // The max value for 1 byte length is 252 From a12fce08da98cc8475f2a90a45973abcbb94cc1a Mon Sep 17 00:00:00 2001 From: Timo Tiuraniemi Date: Fri, 4 Nov 2022 10:07:40 +0200 Subject: [PATCH 086/157] Use upstream hypercore in JS interop tests --- tests/js/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/js/package.json b/tests/js/package.json index 5396f8e6..2c5db7da 100644 --- a/tests/js/package.json +++ b/tests/js/package.json @@ -5,6 +5,6 @@ "step": "node interop.js" }, "dependencies": { - "hypercore": "file:../../../hypercore-js" + "hypercore": "^10" } } From 1e61aff93d4f9ab9415ef04eef3bb1ddf0fae679 Mon Sep 17 00:00:00 2001 From: Timo Tiuraniemi Date: Mon, 21 Nov 2022 14:39:06 +0200 Subject: [PATCH 087/157] Implement missing_nodes, fix bug with searching for byte offset --- src/common/node.rs | 9 ++++- src/core.rs | 31 ++++++++++++++- src/tree/merkle_tree.rs | 88 ++++++++++++++++++++++++++++++++--------- 3 files changed, 107 insertions(+), 21 deletions(-) diff --git a/src/common/node.rs b/src/common/node.rs index 61e5c01b..3dc3f986 100644 --- a/src/common/node.rs +++ b/src/common/node.rs @@ -40,13 +40,20 @@ impl Node { /// Create a new instance. // TODO: ensure sizes are correct. pub fn new(index: u64, hash: Vec, length: u64) -> Self { + let mut blank = true; + for byte in &hash { + if *byte != 0 { + blank = false; + break; + } + } Self { index, hash, length: length as u64, parent: flat_tree::parent(index), data: Some(Vec::with_capacity(0)), - blank: false, + blank, } } diff --git a/src/core.rs b/src/core.rs index b95ef5fe..bcd73149 100644 --- a/src/core.rs +++ b/src/core.rs @@ -474,6 +474,29 @@ where Ok(true) } + /// Used to fill the nodes field of a `RequestBlock` during + /// synchronization. + pub async fn missing_nodes(&mut self, merkle_tree_index: u64) -> Result { + match self.tree.missing_nodes(merkle_tree_index, None)? { + Either::Right(value) => return Ok(value), + Either::Left(instructions) => { + let mut instructions = instructions; + let mut infos: Vec = vec![]; + loop { + infos.extend(self.storage.read_infos_to_vec(&instructions).await?); + match self.tree.missing_nodes(merkle_tree_index, Some(&infos))? { + Either::Right(value) => { + return Ok(value); + } + Either::Left(new_instructions) => { + instructions = new_instructions; + } + } + } + } + } + } + async fn create_valueless_proof( &mut self, block: Option, @@ -915,10 +938,12 @@ mod tests { }, ) .await?; + let index = 6; + let nodes = clone.missing_nodes(index * 2).await?; let proof = main .create_proof( None, - Some(RequestBlock { index: 6, nodes: 0 }), + Some(RequestBlock { index, nodes }), None, Some(RequestUpgrade { start: 0, @@ -936,8 +961,10 @@ mod tests { assert!(clone.get(6).await?.is_none()); // Fetch data for index 6 and verify it is found + let index = 6; + let nodes = clone.missing_nodes(index * 2).await?; let proof = main - .create_proof(Some(RequestBlock { index: 6, nodes: 2 }), None, None, None) + .create_proof(Some(RequestBlock { index, nodes }), None, None, None) .await? .unwrap(); assert!(clone.verify_and_apply_proof(&proof).await?); diff --git a/src/tree/merkle_tree.rs b/src/tree/merkle_tree.rs index d52b62b0..b78089e6 100644 --- a/src/tree/merkle_tree.rs +++ b/src/tree/merkle_tree.rs @@ -195,16 +195,7 @@ impl MerkleTree { infos: Option<&[StoreInfo]>, ) -> Result, u64>> { let index = self.validate_hypercore_index(hypercore_index)?; - // Get nodes out of incoming infos - let nodes: IntMap> = infos_to_nodes(infos); - // Get offset - let offset_result = self.byte_offset_from_nodes(index, &nodes)?; - match offset_result { - Either::Left(offset_instructions) => { - Ok(Either::Left(offset_instructions.into_boxed_slice())) - } - Either::Right(offset) => Ok(Either::Right(offset)), - } + self.byte_offset_from_index(index, infos) } /// Get the byte offset of hypercore index in a changeset @@ -217,8 +208,8 @@ impl MerkleTree { if self.length == hypercore_index { return Ok(Either::Right(self.byte_length)); } - let mut iter = - flat_tree::Iterator::new(hypercore_index_into_merkle_tree_index(hypercore_index)); + let index = hypercore_index_into_merkle_tree_index(hypercore_index); + let mut iter = flat_tree::Iterator::new(index); let mut tree_offset = 0; let mut is_right = false; let mut parent: Option = None; @@ -235,23 +226,23 @@ impl MerkleTree { } } - let search_hypercore_index = if let Some(parent) = parent { + let search_index = if let Some(parent) = parent { let r = changeset .roots .iter() .position(|root| root.index == parent.index); if let Some(r) = r { for i in 0..r { - tree_offset += self.roots[i].length + tree_offset += self.roots[i].length; } return Ok(Either::Right(tree_offset as u64)); } - parent.index / 2 + parent.index } else { - hypercore_index + index }; - match self.byte_offset(search_hypercore_index, infos)? { + match self.byte_offset_from_index(search_index, infos)? { Either::Left(instructions) => Ok(Either::Left(instructions)), Either::Right(offset) => Ok(Either::Right(offset + tree_offset)), } @@ -547,6 +538,41 @@ impl MerkleTree { } } + /// Attempts to get missing nodes from given index. NB: must be called in a loop. + pub fn missing_nodes( + &self, + index: u64, + infos: Option<&[StoreInfo]>, + ) -> Result, u64>> { + let head = 2 * self.length; + let mut iter = flat_tree::Iterator::new(index); + let iter_right_span = iter.index() + iter.factor() / 2 - 1; + + // If the index is not in the current tree, we do not know how many missing nodes there are... + if iter_right_span >= head { + return Ok(Either::Right(0)); + } + + let nodes: IntMap> = infos_to_nodes(infos); + let mut count: u64 = 0; + while !iter.contains(head) { + match self.optional_node(iter.index(), &nodes)? { + Either::Left(instruction) => { + return Ok(Either::Left(vec![instruction].into_boxed_slice())); + } + Either::Right(value) => { + if value.is_none() { + count += 1; + iter.parent(); + } else { + break; + } + } + } + } + Ok(Either::Right(count)) + } + /// Is the changeset commitable to given tree pub fn commitable(&self, changeset: &MerkleTreeChangeset) -> bool { let correct_length: bool = if changeset.upgraded { @@ -641,6 +667,24 @@ impl MerkleTree { Ok(index) } + fn byte_offset_from_index( + &self, + index: u64, + infos: Option<&[StoreInfo]>, + ) -> Result, u64>> { + // Get nodes out of incoming infos + let nodes: IntMap> = infos_to_nodes(infos); + // Get offset + let offset_result = self.byte_offset_from_nodes(index, &nodes)?; + // Get offset + match offset_result { + Either::Left(offset_instructions) => { + Ok(Either::Left(offset_instructions.into_boxed_slice())) + } + Either::Right(offset) => Ok(Either::Right(offset)), + } + } + fn byte_offset_from_nodes( &self, index: u64, @@ -687,6 +731,7 @@ impl MerkleTree { Ok(Either::Left(instructions)) }; } + Err(anyhow!( "Could not calculate byte offset for index {}", index @@ -741,6 +786,13 @@ impl MerkleTree { let result = nodes.get(index); if let Some(node_maybe) = result { if let Some(node) = node_maybe { + if node.blank { + return if allow_miss { + Ok(Either::Right(None)) + } else { + Err(anyhow!("Could not load node: {}", index)) + }; + } return Ok(Either::Right(Some(node.clone()))); } else if allow_miss { return Ok(Either::Right(None)); @@ -749,7 +801,7 @@ impl MerkleTree { } } - // If not, retunr an instruction + // If not, return an instruction let offset = 40 * index; let length = 40; let info = if allow_miss { From b6360e1e0bd7ef34e4da2f9aafa21f965bd8625c Mon Sep 17 00:00:00 2001 From: Timo Tiuraniemi Date: Thu, 12 Jan 2023 16:40:08 +0200 Subject: [PATCH 088/157] Working simple memory benches, improvements in v10 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Unscientfic benchmark shows v9 -> v10 difference in create/read/write: create time: [23.039 µs 23.085 µs 23.138 µs] change: [-9.7528% -9.0070% -8.1511%] (p = 0.00 < 0.05) Performance has improved. Found 11 outliers among 100 measurements (11.00%) 5 (5.00%) low mild 3 (3.00%) high mild 3 (3.00%) high severe write time: [43.521 µs 44.790 µs 45.862 µs] change: [-27.920% -22.938% -17.343%] (p = 0.00 < 0.05) Performance has improved. read time: [939.94 ns 1.0326 µs 1.1560 µs] change: [-46.016% -41.348% -36.380%] (p = 0.00 < 0.05) Performance has improved. Found 13 outliers among 100 measurements (13.00%) 1 (1.00%) low mild 4 (4.00%) high mild 8 (8.00%) high severe --- Cargo.toml | 5 ++++ benches/bench.rs | 58 -------------------------------------- benches/memory.rs | 72 +++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 77 insertions(+), 58 deletions(-) delete mode 100644 benches/bench.rs create mode 100644 benches/memory.rs diff --git a/Cargo.toml b/Cargo.toml index a3bfd502..291059c7 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -54,6 +54,7 @@ remove_dir_all = "0.7.0" tempfile = "3.1.0" async-std = { version = "1.5.0", features = ["attributes"] } sha2 = "0.10.2" +criterion = { version = "0.4", features = ["async_std"] } [features] default = ["v10"] @@ -65,3 +66,7 @@ v10 = ["intmap"] # to verify that this crate works. To run them, use: # cargo test --features js-interop-tests js_interop_tests = [] + +[[bench]] +name = "memory" +harness = false diff --git a/benches/bench.rs b/benches/bench.rs deleted file mode 100644 index 5c113a83..00000000 --- a/benches/bench.rs +++ /dev/null @@ -1,58 +0,0 @@ -#![feature(test)] -extern crate test; - -use anyhow::Error; -use random_access_memory::RandomAccessMemory; -use test::Bencher; - -use hypercore::{Feed, Storage}; - -async fn create_feed(page_size: usize) -> Result, Error> { - let storage = Storage::new( - |_| Box::pin(async move { Ok(RandomAccessMemory::new(page_size)) }), - true, - ) - .await?; - Feed::with_storage(storage).await -} - -#[bench] -fn create(b: &mut Bencher) { - b.iter(|| { - async_std::task::block_on(async { - create_feed(1024).await.unwrap(); - }); - }); -} - -#[bench] -fn write(b: &mut Bencher) { - async_std::task::block_on(async { - let mut feed = create_feed(1024).await.unwrap(); - let data = Vec::from("hello"); - b.iter(|| { - async_std::task::block_on(async { - feed.append(&data).await.unwrap(); - }); - }); - }); -} - -#[bench] -fn read(b: &mut Bencher) { - async_std::task::block_on(async { - let mut feed = create_feed(1024).await.unwrap(); - let data = Vec::from("hello"); - for _ in 0..1000 { - feed.append(&data).await.unwrap(); - } - - let mut i = 0; - b.iter(|| { - async_std::task::block_on(async { - feed.get(i).await.unwrap(); - i += 1; - }); - }); - }); -} diff --git a/benches/memory.rs b/benches/memory.rs new file mode 100644 index 00000000..3eb0148f --- /dev/null +++ b/benches/memory.rs @@ -0,0 +1,72 @@ +use std::time::Instant; + +use anyhow::Error; +use criterion::async_executor::AsyncStdExecutor; +use criterion::{black_box, criterion_group, criterion_main, Criterion}; +#[cfg(feature = "v9")] +use hypercore::{Feed, Storage}; +#[cfg(feature = "v10")] +use hypercore::{Hypercore, Storage}; +use random_access_memory::RandomAccessMemory; + +#[cfg(feature = "v9")] +async fn create_hypercore(page_size: usize) -> Result, Error> { + let storage = Storage::new( + |_| Box::pin(async move { Ok(RandomAccessMemory::new(page_size)) }), + false, + ) + .await?; + Feed::with_storage(storage).await +} + +#[cfg(feature = "v10")] +async fn create_hypercore(page_size: usize) -> Result, Error> { + let storage = Storage::open( + |_| Box::pin(async move { Ok(RandomAccessMemory::new(page_size)) }), + false, + ) + .await?; + Hypercore::new(storage).await +} + +fn create(c: &mut Criterion) { + c.bench_function("create", move |b| { + b.to_async(AsyncStdExecutor).iter(|| create_hypercore(1024)); + }); +} + +fn write(c: &mut Criterion) { + c.bench_function("write", move |b| { + b.to_async(AsyncStdExecutor) + .iter_custom(|iters| async move { + let mut hypercore = create_hypercore(1024).await.unwrap(); + let data = Vec::from("hello"); + let start = Instant::now(); + for _ in 0..iters { + black_box(hypercore.append(&data).await.unwrap()); + } + start.elapsed() + }); + }); +} + +fn read(c: &mut Criterion) { + c.bench_function("read", move |b| { + b.to_async(AsyncStdExecutor) + .iter_custom(|iters| async move { + let mut hypercore = create_hypercore(1024).await.unwrap(); + let data = Vec::from("hello"); + for _ in 0..iters { + hypercore.append(&data).await.unwrap(); + } + let start = Instant::now(); + for i in 0..iters { + black_box(hypercore.get(i).await.unwrap()); + } + start.elapsed() + }); + }); +} + +criterion_group!(benches, create, write, read); +criterion_main!(benches); From a2262b93a889d857df94ba4e054362f8c202c1e6 Mon Sep 17 00:00:00 2001 From: Timo Tiuraniemi Date: Mon, 23 Jan 2023 15:02:47 +0200 Subject: [PATCH 089/157] Added basic tokio compatibility --- Cargo.toml | 13 ++++++++----- tests/js_interop.rs | 9 +++++++-- 2 files changed, 15 insertions(+), 7 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 291059c7..86238d2c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -32,8 +32,8 @@ memory-pager = "0.9.0" merkle-tree-stream = "0.12.0" pretty-hash = "0.4.1" rand = "0.7.3" -random-access-memory = { git = "https://github.com/ttiurani/random-access-memory", branch = "v10" } -random-access-storage = { git = "https://github.com/ttiurani/random-access-storage" , branch = "v10"} +random-access-memory = { git = "https://github.com/ttiurani/random-access-memory", branch = "tokio" } +random-access-storage = { git = "https://github.com/ttiurani/random-access-storage" , branch = "tokio"} sha2 = "0.9.2" sleep-parser = "0.8.0" sparse-bitfield = "0.11.0" @@ -45,23 +45,26 @@ crc32fast = "1.3.2" intmap = { version = "2.0.0", optional = true} [target.'cfg(not(target_arch = "wasm32"))'.dependencies] -random-access-disk = { git = "https://github.com/ttiurani/random-access-disk", branch = "v10" } +random-access-disk = { git = "https://github.com/ttiurani/random-access-disk", branch = "tokio" } [dev-dependencies] quickcheck = "0.9.2" data-encoding = "2.2.0" remove_dir_all = "0.7.0" tempfile = "3.1.0" -async-std = { version = "1.5.0", features = ["attributes"] } +async-std = { version = "1.12.0", features = ["attributes"] } +tokio = { version = "1.24.2", features = ["macros", "rt", "rt-multi-thread"] } sha2 = "0.10.2" criterion = { version = "0.4", features = ["async_std"] } [features] -default = ["v10"] +default = ["v10", "async-std"] # v9 version of the crate v9 = [] # v10 version of the crate v10 = ["intmap"] +tokio = ["random-access-disk/tokio"] +async-std = ["random-access-disk/async-std"] # Used only in interoperability tests under tests/js-interop which use the javascript version of hypercore # to verify that this crate works. To run them, use: # cargo test --features js-interop-tests diff --git a/tests/js_interop.rs b/tests/js_interop.rs index 86d294d6..9f946134 100644 --- a/tests/js_interop.rs +++ b/tests/js_interop.rs @@ -9,6 +9,11 @@ use hypercore::{Hypercore, Storage}; use js::{cleanup, install, js_run_step, prepare_test_set}; use random_access_disk::RandomAccessDisk; +#[cfg(feature = "async-std")] +use async_std::test as async_test; +#[cfg(feature = "tokio")] +use tokio::test as async_test; + const TEST_SET_JS_FIRST: &str = "jsfirst"; const TEST_SET_RS_FIRST: &str = "rsfirst"; @@ -21,7 +26,7 @@ fn init() { }); } -#[async_std::test] +#[async_test] #[cfg_attr(not(feature = "js_interop_tests"), ignore)] #[cfg(feature = "v10")] async fn js_interop_js_first() -> Result<()> { @@ -41,7 +46,7 @@ async fn js_interop_js_first() -> Result<()> { Ok(()) } -#[async_std::test] +#[async_test] #[cfg_attr(not(feature = "js_interop_tests"), ignore)] #[cfg(feature = "v10")] async fn js_interop_rs_first() -> Result<()> { From 4955cc2316f6feb404a196a6ab0134cea44c794a Mon Sep 17 00:00:00 2001 From: Timo Tiuraniemi Date: Mon, 30 Jan 2023 11:21:28 +0200 Subject: [PATCH 090/157] Bump tokio version --- Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index 86238d2c..97c55d9d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -53,7 +53,7 @@ data-encoding = "2.2.0" remove_dir_all = "0.7.0" tempfile = "3.1.0" async-std = { version = "1.12.0", features = ["attributes"] } -tokio = { version = "1.24.2", features = ["macros", "rt", "rt-multi-thread"] } +tokio = { version = "1.25.0", features = ["macros", "rt", "rt-multi-thread"] } sha2 = "0.10.2" criterion = { version = "0.4", features = ["async_std"] } From d7a1210b81e7a3ffd70d708023d5d586e393d5df Mon Sep 17 00:00:00 2001 From: Timo Tiuraniemi Date: Mon, 30 Jan 2023 13:18:16 +0200 Subject: [PATCH 091/157] Remove unnecessary feature from tokio --- Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index 97c55d9d..7c31712f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -53,7 +53,7 @@ data-encoding = "2.2.0" remove_dir_all = "0.7.0" tempfile = "3.1.0" async-std = { version = "1.12.0", features = ["attributes"] } -tokio = { version = "1.25.0", features = ["macros", "rt", "rt-multi-thread"] } +tokio = { version = "1.25.0", default-features = false, features = ["macros", "rt"] } sha2 = "0.10.2" criterion = { version = "0.4", features = ["async_std"] } From 5e4093c8d240f2798592059f32afd063f3b72890 Mon Sep 17 00:00:00 2001 From: Timo Tiuraniemi Date: Fri, 3 Feb 2023 09:02:17 +0200 Subject: [PATCH 092/157] Add support for opening a hypercore without knowing the key pair --- src/core.rs | 25 ++++++++++++++++++++++--- src/oplog/mod.rs | 9 ++++++--- src/storage_v10/mod.rs | 22 +++++++++++++++++----- tests/js_interop.rs | 21 ++++++++++++++------- 4 files changed, 59 insertions(+), 18 deletions(-) diff --git a/src/core.rs b/src/core.rs index bcd73149..9262c329 100644 --- a/src/core.rs +++ b/src/core.rs @@ -47,6 +47,7 @@ pub struct Info { pub byte_length: u64, pub contiguous_length: u64, pub fork: u64, + pub writeable: bool, } impl Hypercore @@ -68,8 +69,21 @@ where /// Creates new hypercore with given storage and (partial) key pair pub async fn new_with_key_pair( - mut storage: Storage, + storage: Storage, key_pair: PartialKeypair, + ) -> Result> { + Hypercore::resume(storage, Some(key_pair)).await + } + + /// Opens an existing hypercore in given storage. + pub async fn open(storage: Storage) -> Result> { + Hypercore::resume(storage, None).await + } + + /// Creates/opens a hypercore with given storage and potentially a key pair + async fn resume( + mut storage: Storage, + key_pair: Option, ) -> Result> { // Open/create oplog let mut oplog_open_outcome = match Oplog::open(&key_pair, None)? { @@ -171,15 +185,19 @@ where } } + let oplog = oplog_open_outcome.oplog; + let header = oplog_open_outcome.header; + let key_pair = header.signer.clone(); + Ok(Hypercore { key_pair, storage, - oplog: oplog_open_outcome.oplog, + oplog, tree, block_store, bitfield, + header, skip_flush_count: 0, - header: oplog_open_outcome.header, }) } @@ -190,6 +208,7 @@ where byte_length: self.tree.byte_length, contiguous_length: self.header.contiguous_length, fork: self.tree.fork, + writeable: self.key_pair.secret.is_some(), } } diff --git a/src/oplog/mod.rs b/src/oplog/mod.rs index 544f8857..64f8d47d 100644 --- a/src/oplog/mod.rs +++ b/src/oplog/mod.rs @@ -84,7 +84,7 @@ const INITIAL_HEADER_BITS: [bool; 2] = [true, false]; impl Oplog { /// Opens an existing Oplog from existing byte buffer or creates a new one. pub fn open( - key_pair: &PartialKeypair, + key_pair: &Option, info: Option, ) -> Result> { match info { @@ -132,9 +132,12 @@ impl Oplog { entries_byte_length: 0, }; OplogOpenOutcome::new(oplog, h2_outcome.state.decode(&existing), Box::new([])) - } else { - // There is nothing in the oplog, start from new. + } else if let Some(key_pair) = key_pair { + // There is nothing in the oplog, start from new given key pair. Self::new(key_pair.clone()) + } else { + // The storage is empty and no key pair given, erroring + return Err(anyhow!("Nothing stored in oplog and key pair not given")); }; // Read headers that might be stored in the existing content diff --git a/src/storage_v10/mod.rs b/src/storage_v10/mod.rs index 99528ddd..f16bca04 100644 --- a/src/storage_v10/mod.rs +++ b/src/storage_v10/mod.rs @@ -58,13 +58,25 @@ where where Cb: Fn(Store) -> std::pin::Pin> + Send>>, { + let mut tree = create(Store::Tree).await?; + let mut data = create(Store::Data).await?; + let mut bitfield = create(Store::Bitfield).await?; + let mut oplog = create(Store::Oplog).await?; + if overwrite { - unimplemented!("Clearing storage not implemented"); + if tree.len().await.map_err(|e| anyhow!(e))? > 0 { + tree.truncate(0).await.map_err(|e| anyhow!(e))?; + } + if data.len().await.map_err(|e| anyhow!(e))? > 0 { + data.truncate(0).await.map_err(|e| anyhow!(e))?; + } + if bitfield.len().await.map_err(|e| anyhow!(e))? > 0 { + bitfield.truncate(0).await.map_err(|e| anyhow!(e))?; + } + if oplog.len().await.map_err(|e| anyhow!(e))? > 0 { + oplog.truncate(0).await.map_err(|e| anyhow!(e))?; + } } - let tree = create(Store::Tree).await?; - let data = create(Store::Data).await?; - let bitfield = create(Store::Bitfield).await?; - let oplog = create(Store::Oplog).await?; let instance = Self { tree, diff --git a/tests/js_interop.rs b/tests/js_interop.rs index 9f946134..9daf082a 100644 --- a/tests/js_interop.rs +++ b/tests/js_interop.rs @@ -68,13 +68,13 @@ async fn js_interop_rs_first() -> Result<()> { #[cfg(feature = "v10")] async fn step_1_create(work_dir: &str) -> Result<()> { - get_hypercore(work_dir).await?; + create_hypercore(work_dir).await?; Ok(()) } #[cfg(feature = "v10")] async fn step_2_append_hello_world(work_dir: &str) -> Result<()> { - let mut hypercore = get_hypercore(work_dir).await?; + let mut hypercore = open_hypercore(work_dir).await?; let append_outcome = hypercore.append_batch(&[b"Hello", b"World"]).await?; assert_eq!(append_outcome.length, 2); assert_eq!(append_outcome.byte_length, 10); @@ -83,7 +83,7 @@ async fn step_2_append_hello_world(work_dir: &str) -> Result<()> { #[cfg(feature = "v10")] async fn step_3_read_and_append_unflushed(work_dir: &str) -> Result<()> { - let mut hypercore = get_hypercore(work_dir).await?; + let mut hypercore = open_hypercore(work_dir).await?; let hello = hypercore.get(0).await?; assert_eq!(hello.unwrap(), b"Hello"); let world = hypercore.get(1).await?; @@ -114,7 +114,7 @@ async fn step_3_read_and_append_unflushed(work_dir: &str) -> Result<()> { #[cfg(feature = "v10")] async fn step_4_append_with_flush(work_dir: &str) -> Result<()> { - let mut hypercore = get_hypercore(work_dir).await?; + let mut hypercore = open_hypercore(work_dir).await?; for i in 0..5 { let append_outcome = hypercore.append(&[i]).await?; assert_eq!(append_outcome.length, (6 + i + 1) as u64); @@ -125,7 +125,7 @@ async fn step_4_append_with_flush(work_dir: &str) -> Result<()> { #[cfg(feature = "v10")] async fn step_5_clear_some(work_dir: &str) -> Result<()> { - let mut hypercore = get_hypercore(work_dir).await?; + let mut hypercore = open_hypercore(work_dir).await?; hypercore.clear(5, 6).await?; hypercore.clear(7, 9).await?; let info = hypercore.info(); @@ -144,13 +144,20 @@ async fn step_5_clear_some(work_dir: &str) -> Result<()> { } #[cfg(feature = "v10")] -async fn get_hypercore(work_dir: &str) -> Result> { +async fn create_hypercore(work_dir: &str) -> Result> { let path = Path::new(work_dir).to_owned(); let key_pair = get_test_key_pair(); - let storage = Storage::new_disk(&path, false).await?; + let storage = Storage::new_disk(&path, true).await?; Ok(Hypercore::new_with_key_pair(storage, key_pair).await?) } +#[cfg(feature = "v10")] +async fn open_hypercore(work_dir: &str) -> Result> { + let path = Path::new(work_dir).to_owned(); + let storage = Storage::new_disk(&path, false).await?; + Ok(Hypercore::open(storage).await?) +} + fn step_0_hash() -> common::HypercoreHash { common::HypercoreHash { bitfield: None, From 499a8f3c8766b94f5b1627e1c8252a02b818e78a Mon Sep 17 00:00:00 2001 From: Timo Tiuraniemi Date: Fri, 17 Feb 2023 11:16:58 +0200 Subject: [PATCH 093/157] Set merkle tree signature on open from header. Fixes bug with creating proof on a newly opened hypercore failing --- src/tree/merkle_tree.rs | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/tree/merkle_tree.rs b/src/tree/merkle_tree.rs index b78089e6..8d8f1423 100644 --- a/src/tree/merkle_tree.rs +++ b/src/tree/merkle_tree.rs @@ -3,6 +3,7 @@ use anyhow::{anyhow, ensure}; use ed25519_dalek::Signature; use futures::future::Either; use intmap::IntMap; +use std::convert::TryFrom; use crate::common::NodeByteRange; use crate::common::{Proof, ValuelessProof}; @@ -83,6 +84,11 @@ impl MerkleTree { if length > 0 { length = length / 2; } + let signature: Option = if header_tree.signature.len() > 0 { + Some(Signature::try_from(&*header_tree.signature)?) + } else { + None + }; Ok(Either::Right(Self { roots, @@ -92,7 +98,7 @@ impl MerkleTree { unflushed: IntMap::new(), truncated: false, truncate_to: 0, - signature: None, + signature, })) } } From 127df9ed89b39038537ccb69ba2891e8e21d3c9a Mon Sep 17 00:00:00 2001 From: Timo Tiuraniemi Date: Tue, 14 Mar 2023 12:55:33 +0200 Subject: [PATCH 094/157] Enabled quickcheck model test for v10 as well --- Cargo.toml | 2 +- tests/model.rs | 106 +++++++++++++++++++++++++++++++++++++++++++++++-- 2 files changed, 104 insertions(+), 4 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 7c31712f..4178909c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -53,7 +53,7 @@ data-encoding = "2.2.0" remove_dir_all = "0.7.0" tempfile = "3.1.0" async-std = { version = "1.12.0", features = ["attributes"] } -tokio = { version = "1.25.0", default-features = false, features = ["macros", "rt"] } +tokio = { version = "1.25.0", default-features = false, features = ["macros", "rt", "rt-multi-thread"] } sha2 = "0.10.2" criterion = { version = "0.4", features = ["async_std"] } diff --git a/tests/model.rs b/tests/model.rs index bf43419f..bf84acad 100644 --- a/tests/model.rs +++ b/tests/model.rs @@ -11,9 +11,19 @@ const MAX_FILE_SIZE: u64 = 5 * 10; // 5mb #[derive(Clone, Debug)] enum Op { - Get { index: u64 }, - Append { data: Vec }, + Get { + index: u64, + }, + Append { + data: Vec, + }, + #[cfg(feature = "v9")] Verify, + #[cfg(feature = "v10")] + Clear { + len_divisor_for_start: u8, + len_divisor_for_length: u8, + }, } impl Arbitrary for Op { @@ -32,14 +42,25 @@ impl Arbitrary for Op { } Op::Append { data } } + #[cfg(feature = "v9")] 2 => Op::Verify, + #[cfg(feature = "v10")] + 2 => { + let len_divisor_for_start: u8 = g.gen_range(1, 17); + let len_divisor_for_length: u8 = g.gen_range(1, 17); + Op::Clear { + len_divisor_for_start, + len_divisor_for_length, + } + } err => panic!("Invalid choice {}", err), } } } -#[cfg(feature = "v9")] quickcheck! { + + #[cfg(feature = "v9")] fn implementation_matches_model(ops: Vec) -> bool { async_std::task::block_on(async { let page_size = 50; @@ -79,4 +100,83 @@ quickcheck! { true }) } + #[cfg(all(feature = "v10", feature = "async-std"))] + fn implementation_matches_model(ops: Vec) -> bool { + async_std::task::block_on(assert_implementation_matches_model(ops)) + } + + #[cfg(all(feature = "v10", feature = "tokio"))] + fn implementation_matches_model(ops: Vec) -> bool { + let rt = tokio::runtime::Runtime::new().unwrap(); + rt.block_on(async { + assert_implementation_matches_model(ops).await + }) + } +} + +#[cfg(feature = "v10")] +async fn assert_implementation_matches_model(ops: Vec) -> bool { + use hypercore::{Hypercore, Storage}; + + let storage = Storage::new_memory() + .await + .expect("Memory storage creation should be successful"); + let mut hypercore = Hypercore::new(storage) + .await + .expect("Hypercore creation should be successful"); + + let mut model: Vec>> = vec![]; + + for op in ops { + match op { + Op::Append { data } => { + hypercore + .append(&data) + .await + .expect("Append should be successful"); + model.push(Some(data)); + } + Op::Get { index } => { + let data = hypercore + .get(index) + .await + .expect("Get should be successful"); + if index >= hypercore.info().length { + assert_eq!(data, None); + } else { + assert_eq!(data, model[index as usize].clone()); + } + } + Op::Clear { + len_divisor_for_start, + len_divisor_for_length, + } => { + let start = { + let result = model.len() as u64 / len_divisor_for_start as u64; + if result == model.len() as u64 { + if model.len() > 0 { + result - 1 + } else { + 0 + } + } else { + result + } + }; + let length = model.len() as u64 / len_divisor_for_length as u64; + let end = start + length; + let model_end = if end < model.len() as u64 { + end + } else { + model.len() as u64 + }; + hypercore + .clear(start, end) + .await + .expect("Clear should be successful"); + model[start as usize..model_end as usize].fill(None); + } + } + } + true } From 0ce0e850a6556034851c6d7d8ef8dea688d1ec98 Mon Sep 17 00:00:00 2001 From: Timo Tiuraniemi Date: Fri, 17 Mar 2023 09:15:08 +0200 Subject: [PATCH 095/157] Remove v9/v10 feature flags and v9 code --- Cargo.toml | 8 +- benches/memory.rs | 14 - examples/async.rs | 35 -- examples/main.rs | 33 -- src/audit.rs | 20 -- src/bitfield/iterator.rs | 158 --------- src/bitfield/masks.rs | 108 ------ src/bitfield/mod.rs | 378 --------------------- src/common/node.rs | 50 --- src/crypto/hash.rs | 8 - src/crypto/mod.rs | 1 - src/feed.rs | 677 -------------------------------------- src/feed_builder.rs | 90 ----- src/lib.rs | 55 ---- src/prelude.rs | 15 - src/proof.rs | 30 -- src/replicate/message.rs | 6 - src/replicate/mod.rs | 5 - src/replicate/peer.rs | 40 --- src/storage/mod.rs | 444 ------------------------- src/storage/persist.rs | 19 -- tests/bitfield.rs | 204 ------------ tests/common/mod.rs | 16 +- tests/compact_encoding.rs | 2 - tests/compat.rs | 188 ----------- tests/core.rs | 7 +- tests/feed.rs | 358 -------------------- tests/js_interop.rs | 14 +- tests/model.rs | 56 +--- tests/regression.rs | 20 -- tests/storage.rs | 58 ---- 31 files changed, 11 insertions(+), 3106 deletions(-) delete mode 100644 src/audit.rs delete mode 100644 src/bitfield/iterator.rs delete mode 100644 src/bitfield/masks.rs delete mode 100644 src/bitfield/mod.rs delete mode 100644 src/feed.rs delete mode 100644 src/feed_builder.rs delete mode 100644 src/proof.rs delete mode 100644 src/replicate/message.rs delete mode 100644 src/replicate/mod.rs delete mode 100644 src/replicate/peer.rs delete mode 100644 src/storage/mod.rs delete mode 100644 src/storage/persist.rs delete mode 100644 tests/bitfield.rs delete mode 100644 tests/compat.rs delete mode 100644 tests/feed.rs delete mode 100644 tests/regression.rs delete mode 100644 tests/storage.rs diff --git a/Cargo.toml b/Cargo.toml index 4178909c..3a840125 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -42,7 +42,7 @@ bitfield-rle = "0.2.0" futures = "0.3.4" async-std = "1.5.0" crc32fast = "1.3.2" -intmap = { version = "2.0.0", optional = true} +intmap = "2.0.0" [target.'cfg(not(target_arch = "wasm32"))'.dependencies] random-access-disk = { git = "https://github.com/ttiurani/random-access-disk", branch = "tokio" } @@ -58,11 +58,7 @@ sha2 = "0.10.2" criterion = { version = "0.4", features = ["async_std"] } [features] -default = ["v10", "async-std"] -# v9 version of the crate -v9 = [] -# v10 version of the crate -v10 = ["intmap"] +default = ["async-std"] tokio = ["random-access-disk/tokio"] async-std = ["random-access-disk/async-std"] # Used only in interoperability tests under tests/js-interop which use the javascript version of hypercore diff --git a/benches/memory.rs b/benches/memory.rs index 3eb0148f..7d2a89d5 100644 --- a/benches/memory.rs +++ b/benches/memory.rs @@ -3,23 +3,9 @@ use std::time::Instant; use anyhow::Error; use criterion::async_executor::AsyncStdExecutor; use criterion::{black_box, criterion_group, criterion_main, Criterion}; -#[cfg(feature = "v9")] -use hypercore::{Feed, Storage}; -#[cfg(feature = "v10")] use hypercore::{Hypercore, Storage}; use random_access_memory::RandomAccessMemory; -#[cfg(feature = "v9")] -async fn create_hypercore(page_size: usize) -> Result, Error> { - let storage = Storage::new( - |_| Box::pin(async move { Ok(RandomAccessMemory::new(page_size)) }), - false, - ) - .await?; - Feed::with_storage(storage).await -} - -#[cfg(feature = "v10")] async fn create_hypercore(page_size: usize) -> Result, Error> { let storage = Storage::open( |_| Box::pin(async move { Ok(RandomAccessMemory::new(page_size)) }), diff --git a/examples/async.rs b/examples/async.rs index 2f9ce57a..f0610c2c 100644 --- a/examples/async.rs +++ b/examples/async.rs @@ -1,38 +1,3 @@ -use async_std::task; -#[cfg(feature = "v9")] -use hypercore::Feed; -use random_access_storage::RandomAccess; -use std::fmt::Debug; - -#[cfg(feature = "v9")] -async fn append(feed: &mut Feed, content: &[u8]) -where - T: RandomAccess> + Debug + Send, -{ - feed.append(content).await.unwrap(); -} - -#[cfg(feature = "v9")] -async fn print(feed: &mut Feed) -where - T: RandomAccess> + Debug + Send, -{ - println!("{:?}", feed.get(0).await); - println!("{:?}", feed.get(1).await); -} - -#[cfg(feature = "v9")] -fn main() { - task::block_on(task::spawn(async { - let mut feed = Feed::default(); - - append(&mut feed, b"hello").await; - append(&mut feed, b"world").await; - print(&mut feed).await; - })); -} - -#[cfg(feature = "v10")] fn main() { unimplemented!() } diff --git a/examples/main.rs b/examples/main.rs index e5000173..d7f45deb 100644 --- a/examples/main.rs +++ b/examples/main.rs @@ -1,37 +1,4 @@ -#[cfg(feature = "v9")] -use hypercore::Feed; - #[async_std::main] -#[cfg(feature = "v9")] -async fn main() { - let mut feed = Feed::open("feed.db").await.expect("Failed to create dir"); - - feed.append(b"hello").await.unwrap(); - feed.append(b"world").await.unwrap(); - - drop(feed); - - let mut feed = Feed::open("feed.db").await.expect("Failed to create dir"); - - feed.append(b"welcome").await.unwrap(); - feed.append(b"back").await.unwrap(); - - println!("{:?}", format_res(feed.get(0).await)); // prints "hello" - println!("{:?}", format_res(feed.get(1).await)); // prints "world" - println!("{:?}", format_res(feed.get(2).await)); // prints "welcome" - println!("{:?}", format_res(feed.get(3).await)); // prints "back" -} - -#[async_std::main] -#[cfg(feature = "v10")] async fn main() { unimplemented!(); } - -fn format_res(res: anyhow::Result>>) -> String { - match res { - Ok(Some(bytes)) => String::from_utf8(bytes).expect("Shouldnt fail in example"), - Ok(None) => "Got None in feed".to_string(), - Err(e) => format!("Error getting value from feed, reason = {}", e), - } -} diff --git a/src/audit.rs b/src/audit.rs deleted file mode 100644 index ef6f9a43..00000000 --- a/src/audit.rs +++ /dev/null @@ -1,20 +0,0 @@ -/// The audit report for a feed, created by the `.audit()` method. -#[derive(Debug, PartialEq, Clone)] -pub struct Audit { - /// The number of valid blocks identified - pub valid_blocks: u64, - /// The number of invalid blocks identified - pub invalid_blocks: u64, -} - -impl Audit { - /// Access the `valid_blocks` field from the proof. - pub fn valid_blocks(&self) -> u64 { - self.valid_blocks - } - - /// Access the `invalid_blocks` field from the proof. - pub fn invalid_blocks(&self) -> u64 { - self.invalid_blocks - } -} diff --git a/src/bitfield/iterator.rs b/src/bitfield/iterator.rs deleted file mode 100644 index 2c6d3d9a..00000000 --- a/src/bitfield/iterator.rs +++ /dev/null @@ -1,158 +0,0 @@ -//! Iterate over a bitfield. - -use super::Bitfield; - -/// Iterate over a bitfield. -#[derive(Debug)] -pub struct Iterator<'a> { - start: u64, - end: u64, - index_end: u64, - pos: Option, - byte: u8, - bitfield: &'a mut Bitfield, -} - -impl<'a> Iterator<'a> { - /// Create a new instance. - pub fn new(bitfield: &'a mut Bitfield) -> Self { - Self { - start: 0, - end: 0, - index_end: 0, - pos: Some(0), - byte: 0, - bitfield, - } - } - - /// Grow the bitfield if needed. - pub fn range(&mut self, start: u64, end: u64) { - self.start = start; - self.end = end; - self.index_end = 2 * ((end + 31) / 32); - - if self.end > self.bitfield.length { - self.bitfield.expand(self.end); - } - } - - /// Seek to `offset` - pub fn seek(&mut self, mut offset: u64) -> &mut Self { - offset += self.start; - // FIXME This is fishy. Offset and start is unsigned, so `offset < self.start` can only - // be true when the previous addition overflows. The overflow would cause a panic, so, - // either the addition should be a wrapping_add, or rather, the original offset should - // be checked to ensure it is less than `self.end - self.start`. - if offset < self.start { - offset = self.start; - } - - if offset >= self.end { - self.pos = None; - return self; - } - - let o = offset % 8; - - let pos = offset / 8; - self.pos = Some(pos); - - self.byte = self.bitfield.data.get_byte(pos as usize) - | self.bitfield.masks.data_iterate[o as usize]; - - self - } - - pub fn next(&mut self) -> Option { - let mut pos = self.pos?; - - let mut free = self.bitfield.masks.next_data_0_bit[self.byte as usize]; - - while free == -1 { - pos += 1; - self.byte = self.bitfield.data.get_byte(pos as usize); - free = self.bitfield.masks.next_data_0_bit[self.byte as usize]; - - if free == -1 { - pos = self.skip_ahead(pos)?; - - self.byte = self.bitfield.data.get_byte(pos as usize); - free = self.bitfield.masks.next_data_0_bit[self.byte as usize]; - } - } - self.pos = Some(pos); - - self.byte |= self.bitfield.masks.data_iterate[free as usize + 1]; - - let n = 8 * pos + free as u64; - if n < self.end { - Some(n) - } else { - None - } - } - - pub fn skip_ahead(&mut self, start: u64) -> Option { - let bitfield_index = &self.bitfield.index; - let tree_end = self.index_end; - let iter = &mut self.bitfield.iterator; - let o = start & 3; - - iter.seek(2 * (start / 4)); - - let mut tree_byte = bitfield_index.get_byte(iter.index() as usize) - | self.bitfield.masks.index_iterate[o as usize]; - - while self.bitfield.masks.next_index_0_bit[tree_byte as usize] == -1 { - if iter.is_left() { - iter.next(); - } else { - iter.next(); - iter.parent(); - } - - if right_span(iter) >= tree_end { - while right_span(iter) >= tree_end && is_parent(iter) { - iter.left_child(); - } - if right_span(iter) >= tree_end { - return None; - } - } - - tree_byte = bitfield_index.get_byte(iter.index() as usize); - } - - while iter.factor() > 2 { - if self.bitfield.masks.next_index_0_bit[tree_byte as usize] < 2 { - iter.left_child(); - } else { - iter.right_child(); - } - - tree_byte = bitfield_index.get_byte(iter.index() as usize); - } - - let mut free = self.bitfield.masks.next_index_0_bit[tree_byte as usize]; - if free == -1 { - free = 4; - } - - let next = iter.index() * 2 + free as u64; - - if next <= start { - Some(start + 1) - } else { - Some(next) - } - } -} - -fn right_span(iter: &flat_tree::Iterator) -> u64 { - iter.index() + iter.factor() / 2 - 1 -} - -fn is_parent(iter: &flat_tree::Iterator) -> bool { - iter.index() & 1 == 1 -} diff --git a/src/bitfield/masks.rs b/src/bitfield/masks.rs deleted file mode 100644 index 2f9f0704..00000000 --- a/src/bitfield/masks.rs +++ /dev/null @@ -1,108 +0,0 @@ -//! Masks used to determine how to update bytes. -//! -//! This piece of code is still a bit unclear; lots of magic numbers. It'd be -//! good to figure out what things mean. - -#[derive(Debug)] -pub(super) struct Masks { - pub index_update: Vec, - pub index_iterate: Vec, - pub data_iterate: Vec, - pub data_update: Vec, - pub map_parent_right: Vec, - pub map_parent_left: Vec, - pub next_data_0_bit: Vec, - pub next_index_0_bit: Vec, - pub total_1_bits: Vec, -} - -// Masks are underscored at every 8 bytes. -impl Masks { - pub fn new() -> Self { - let index_update = vec![ - 0b00_11_11_11, // 63 - 0b11_00_11_11, // 207 - 0b11_11_00_11, // 243 - 0b11_11_11_00, // 252 - ]; - - let index_iterate = vec![ - 0b00_00_00_00, // 0 - 0b11_00_00_00, // 192 - 0b11_11_00_00, // 240 - 0b11_11_11_00, // 252 - ]; - - let data_iterate = vec![ - 0b00_00_00_00, // 0 - 0b10_00_00_00, // 128 - 0b11_00_00_00, // 192 - 0b11_10_00_00, // 224 - 0b11_11_00_00, // 240 - 0b11_11_10_00, // 248 - 0b11_11_11_00, // 252 - 0b11_11_11_10, // 254 - 0b11_11_11_11, // 255 - ]; - - let data_update = vec![ - 0b01_11_11_11, // 127 - 0b10_11_11_11, // 191 - 0b11_01_11_11, // 223 - 0b11_10_11_11, // 239 - 0b11_11_01_11, // 247 - 0b11_11_10_11, // 251 - 0b11_11_11_01, // 253 - 0b11_11_11_10, // 254 - ]; - - let mut map_parent_right = vec![0; 256]; - let mut map_parent_left = vec![0; 256]; - - for i in 0..256 { - let a = (i & (15 << 4)) >> 4; - let b = i & 15; - - let left = if a == 15 { - 3 - } else if a == 0 { - 0 - } else { - 1 - }; - - let right = if b == 15 { - 3 - } else if b == 0 { - 0 - } else { - 1 - }; - - map_parent_right[i] = left | right; - map_parent_left[i] = map_parent_right[i] << 4; - } - - let total_1_bits: Vec<_> = (0..256).map(|n| (n as u8).count_ones() as u8).collect(); - - let mut next_data_0_bit: Vec<_> = (0..256) - .map(|n| (!n as u8).leading_zeros() as i16) - .collect(); - next_data_0_bit[255] = -1; - - let mut next_index_0_bit: Vec<_> = next_data_0_bit.iter().map(|n| n / 2).collect(); - next_index_0_bit[255] = -1; - - Self { - index_update, - index_iterate, - data_iterate, - data_update, - map_parent_right, - map_parent_left, - next_data_0_bit, - next_index_0_bit, - total_1_bits, - } - } -} diff --git a/src/bitfield/mod.rs b/src/bitfield/mod.rs deleted file mode 100644 index 9546a1e2..00000000 --- a/src/bitfield/mod.rs +++ /dev/null @@ -1,378 +0,0 @@ -//! Bitfield module. Exposes `{data, tree, index}` internally. Serializable to -//! disk. -//! -//! TODO(yw): Document the magic mask format. (Will help to look at binary -//! versions of the numbers). -//! -//! TODO(yw): Document the use cases for this module, especially when opposed to -//! `sparse_bitfield`. -//! -//! NOTE(yw): in the JavaScript version, this code uses a single pager under the -//! hood. Because of Rust's borrow rules, that would be tricky to pull off for -//! us. So instead we've chosen to create three separate instances, with three -//! separate pagers powering it. -//! This means that when serializing to disk, we need to weave the contents of -//! all three of the pagers into a single instance. And when restoring it from -//! disk, we must do so again. -//! We need to make sure the performance impact of this stays well within -//! bounds. - -mod iterator; -mod masks; - -use self::masks::Masks; -use flat_tree::{self, Iterator as FlatIterator}; -pub use sparse_bitfield::{Bitfield as SparseBitfield, Change}; -use std::ops::Range; - -/// Bitfield with `{data, tree, index} fields.` -#[derive(Debug)] -pub struct Bitfield { - data: SparseBitfield, - index: SparseBitfield, - #[allow(dead_code)] - page_len: u64, - length: u64, - masks: Masks, - iterator: FlatIterator, -} - -impl Bitfield { - /// Create a new instance. - pub fn new() -> (Self, SparseBitfield) { - let s = Self { - data: SparseBitfield::new(1024), - index: SparseBitfield::new(256), - page_len: 3328, - length: 0, - masks: Masks::new(), - iterator: FlatIterator::new(0), - }; - (s, SparseBitfield::new(2048)) - } - - /// Create new instance from byteslice - pub fn from_slice(slice: &[u8]) -> (Self, SparseBitfield) { - // khodzha: - // slice is packed as data|tree|index|data|tree|index|... - // so for each 1024 + 2048 + 256 bytes - // we extract first 1024 bytes to data - // then next 2048 bytes to tree - // then next 256 bytes to index - let mut data = SparseBitfield::new(1024); - let mut tree = SparseBitfield::new(2048); - let mut index = SparseBitfield::new(256); - slice - .chunks_exact(1024 + 2048 + 256) - .enumerate() - .for_each(|(page_idx, chunk)| { - chunk.iter().enumerate().for_each(|(idx, byte)| { - if idx < 1024 { - data.set_byte(page_idx * 1024 + idx, *byte); - } else if idx < 1024 + 2048 { - tree.set_byte(page_idx * 1024 + (idx - 1024), *byte); - } else { - index.set_byte(page_idx * 1024 + (idx - 1024 - 2048), *byte); - } - }); - }); - let length = data.len() as u64; - let s = Self { - data, - index, - length, - page_len: 3328, - masks: Masks::new(), - iterator: FlatIterator::new(0), - }; - - (s, tree) - } - - /// Convert to vec - pub fn to_bytes(&self, tree: &tree_index::TreeIndex) -> std::io::Result> { - let tree = tree.as_bitfield(); - let data_bytes = self.data.to_bytes()?; - let tree_bytes = tree.to_bytes()?; - let index_bytes = self.index.to_bytes()?; - - let max_pages_len = std::cmp::max( - std::cmp::max(self.data.page_len(), tree.page_len()), - self.index.page_len(), - ); - - let data_ps = self.data.page_size(); - let tree_ps = tree.page_size(); - let index_ps = self.index.page_size(); - - let total_ps = data_ps + tree_ps + index_ps; - - let mut vec = Vec::with_capacity(max_pages_len * total_ps); - - for i in 0..max_pages_len { - extend_buf_from_slice(&mut vec, &data_bytes, i, data_ps); - extend_buf_from_slice(&mut vec, &tree_bytes, i, tree_ps); - extend_buf_from_slice(&mut vec, &index_bytes, i, index_ps); - } - - Ok(vec) - } - - /// Get the current length - pub fn len(&self) -> u64 { - self.length - } - - /// Returns `true` if the bitfield is empty - pub fn is_empty(&self) -> bool { - self.length == 0 - } - - /// Set a value at an index. - pub fn set(&mut self, index: u64, value: bool) -> Change { - let o = mask_8b(index); - let index = (index - o) / 8; - - let value = if value { - self.data.get_byte(index as usize) | 128 >> o - } else { - self.data.get_byte(index as usize) & self.masks.data_update[o as usize] - }; - - if self.data.set_byte(index as usize, value).is_unchanged() { - return Change::Unchanged; - } - - self.length = self.data.len() as u64; - self.set_index(index, value); - Change::Changed - } - - /// Get a value at a position in the bitfield. - pub fn get(&mut self, index: u64) -> bool { - self.data.get(index as usize) - } - - /// Calculate the total for the whole data. - pub fn total(&mut self) -> u8 { - let len = self.data.len() as u64; - self.total_with_range(0..len) - } - - /// Calculate the total of ... TODO(yw) - pub fn total_with_start(&mut self, start: u64) -> u8 { - let len = self.data.len() as u64; - self.total_with_range(start..len) - } - - /// Calculate the total of ... TODO(yw) - pub fn total_with_range(&mut self, range: Range) -> u8 { - let start = range.start; - let end = range.end; - - if end < start { - return 0; - } - - if end > self.data.len() as u64 { - self.expand(end); - } - - let o = mask_8b(start); - let e = mask_8b(end); - - let pos = (start - o) / 8; - let last = (end - e) / 8; - - let left_mask = 255 - self.masks.data_iterate[o as usize]; - let right_mask = self.masks.data_iterate[e as usize]; - - let byte = self.data.get_byte(pos as usize); - if pos == last { - let index = (byte & left_mask & right_mask) as u64; - return self.masks.total_1_bits[index as usize]; - } - let index = (byte & left_mask) as u64; - let mut total = self.masks.total_1_bits[index as usize]; - - for i in pos + 1..last { - let index = self.data.get_byte(i as usize) as u64; - total += self.masks.total_1_bits[index as usize]; - } - - let index: u64 = self.data.get_byte(last as usize) as u64 & right_mask as u64; - total + self.masks.total_1_bits[index as usize] - } - - /// Set a value at index. - /// - ///```txt - /// (a + b | c + d | e + f | g + h) - /// -> (a | b | c | d) (e | f | g | h) - ///``` - /// - /// NOTE(yw): lots of magic values going on; I have no idea what we're doing - /// here. - fn set_index(&mut self, mut index: u64, value: u8) -> Change { - let o = index & 3; - index = (index - o) / 4; - - let start = tree_index(index); - - let left = self.index.get_byte(start as usize) & self.masks.index_update[o as usize]; - let right = get_index_value(value) >> tree_index(o); - let mut byte = left | right; - let len = self.index.len(); - let max_len = self.data.page_len() * 256; - - self.iterator.seek(start); - - while self.iterator.index() < max_len as u64 - && self - .index - .set_byte(self.iterator.index() as usize, byte) - .is_changed() - { - if self.iterator.is_left() { - let index: u64 = self.index.get_byte(self.iterator.sibling() as usize).into(); - byte = self.masks.map_parent_left[byte as usize] - | self.masks.map_parent_right[index as usize]; - } else { - let index: u64 = self - .index - .get_byte(self.iterator.sibling() as usize) // FIXME: out of bounds read - .into(); - byte = self.masks.map_parent_right[byte as usize] - | self.masks.map_parent_left[index as usize]; - } - self.iterator.parent(); - } - - if len != self.index.len() { - self.expand(len as u64); - } - - if self.iterator.index() == start { - Change::Unchanged - } else { - Change::Changed - } - } - - fn expand(&mut self, len: u64) { - let mut roots = vec![]; // FIXME: alloc. - flat_tree::full_roots(tree_index(len), &mut roots); - let bf = &mut self.index; - let ite = &mut self.iterator; - let masks = &self.masks; - let mut byte; - - for root in roots { - ite.seek(root); - byte = bf.get_byte(ite.index() as usize); - - loop { - if ite.is_left() { - let index = bf.get_byte(ite.sibling() as usize) as u64; - byte = masks.map_parent_left[byte as usize] - | masks.map_parent_right[index as usize]; - } else { - let index = bf.get_byte(ite.sibling() as usize) as u64; - byte = masks.map_parent_right[byte as usize] - | masks.map_parent_left[index as usize]; - } - - if set_byte_no_alloc(bf, ite.parent(), byte).is_unchanged() { - break; - } - } - } - } - - // TODO: use the index to speed this up *a lot* - /// https://github.com/mafintosh/hypercore/blob/06f3a1f573cb74ee8cfab2742455318fbf7cc3a2/lib/bitfield.js#L111-L126 - pub fn compress(&self, start: usize, length: usize) -> std::io::Result> { - // On Node versions this fields might not be present on the want/request message - // When both start and length are not present (!0 in node is false), return all data bytes encoded - if start == 0 && length == 0 { - return Ok(bitfield_rle::encode(&self.data.to_bytes()?)); - } - - use std::io::{Cursor, Write}; - let mut buf = Cursor::new(Vec::with_capacity(length)); - - let page_size = self.data.page_size() as f64; - let mut p = start as f64 / page_size / 8.0; - let end = p + length as f64 / page_size / 8.0; - let offset = p * page_size; - - while p < end { - let index = p as usize; - let page = self.data.pages.get(index); - if let Some(page) = page { - if page.len() != 0 { - buf.set_position((p * page_size - offset) as u64); - buf.write_all(&page)?; - } - } - p += 1.0; - } - - Ok(bitfield_rle::encode(&buf.into_inner())) - } - - /// Constructs an iterator from start to end - pub fn iterator(&mut self) -> iterator::Iterator<'_> { - let len = self.length; - self.iterator_with_range(0, len) - } - - /// Constructs an iterator from `start` to `end` - pub fn iterator_with_range(&mut self, start: u64, end: u64) -> iterator::Iterator<'_> { - let mut iter = iterator::Iterator::new(self); - iter.range(start, end); - iter.seek(0); - - iter - } -} - -// NOTE: can we move this into `sparse_bitfield`? -fn set_byte_no_alloc(bf: &mut SparseBitfield, index: u64, byte: u8) -> Change { - if 8 * index >= bf.len() as u64 { - return Change::Unchanged; - } - bf.set_byte(index as usize, byte) -} - -#[inline] -fn get_index_value(index: u8) -> u8 { - match index { - 255 => 192, - 0 => 0, - _ => 64, - } -} - -#[inline] -fn mask_8b(num: u64) -> u64 { - num & 7 -} - -/// Convert the index to the index in the tree. -#[inline] -fn tree_index(index: u64) -> u64 { - 2 * index -} - -// copies slice to buf or fills buf with len-of-slice zeros -fn extend_buf_from_slice(buf: &mut Vec, bytes: &[u8], i: usize, pagesize: usize) { - if i * pagesize >= bytes.len() { - for _ in 0..pagesize { - buf.push(0); - } - } else { - let range = (i * pagesize)..((i + 1) * pagesize); - buf.extend_from_slice(&bytes[range]); - } -} diff --git a/src/common/node.rs b/src/common/node.rs index 3dc3f986..12b973bd 100644 --- a/src/common/node.rs +++ b/src/common/node.rs @@ -1,17 +1,9 @@ -#[cfg(feature = "v9")] // tree module has the byte manipulation for v10 -use anyhow::ensure; -#[cfg(feature = "v9")] // tree module has the byte manipulation for v10 -use anyhow::Result; -#[cfg(feature = "v9")] // tree module has the byte manipulation for v10 -use byteorder::{BigEndian, ReadBytesExt, WriteBytesExt}; use merkle_tree_stream::Node as NodeTrait; use merkle_tree_stream::{NodeKind, NodeParts}; use pretty_hash::fmt as pretty_fmt; use std::cmp::Ordering; use std::convert::AsRef; use std::fmt::{self, Display}; -#[cfg(feature = "v9")] // tree module has the byte manipulation for v10 -use std::io::Cursor; use crate::crypto::Hash; @@ -68,48 +60,6 @@ impl Node { blank: true, } } - - /// Convert a vector to a new instance. - /// - /// Requires the index at which the buffer was read to be passed. - #[cfg(feature = "v9")] // tree module has the byte manipulation for v10 - pub fn from_bytes(index: u64, buffer: &[u8]) -> Result { - ensure!(buffer.len() == 40, "buffer should be 40 bytes"); - - let parent = flat_tree::parent(index); - let mut reader = Cursor::new(buffer); - - // TODO: subslice directly, move cursor forward. - let capacity = 32; - let mut hash = Vec::with_capacity(capacity); - let mut blank = true; - for _ in 0..capacity { - let byte = reader.read_u8()?; - if blank && byte != 0 { - blank = false; - }; - hash.push(byte); - } - - let length = reader.read_u64::()?; - Ok(Self { - hash, - length, - index, - parent, - data: Some(Vec::with_capacity(0)), - blank, - }) - } - - /// Convert to a buffer that can be written to disk. - #[cfg(feature = "v9")] // tree module has the byte manipulation for v10 - pub fn to_bytes(&self) -> Result> { - let mut writer = Vec::with_capacity(40); - writer.extend_from_slice(&self.hash); - writer.write_u64::(self.length as u64)?; - Ok(writer) - } } impl NodeTrait for Node { diff --git a/src/crypto/hash.rs b/src/crypto/hash.rs index 4688a689..c33417e9 100644 --- a/src/crypto/hash.rs +++ b/src/crypto/hash.rs @@ -1,7 +1,6 @@ pub use blake2_rfc::blake2b::Blake2bResult; use crate::common::Node; -#[cfg(feature = "v10")] use crate::compact_encoding::State; use blake2_rfc::blake2b::Blake2b; use byteorder::{BigEndian, WriteBytesExt}; @@ -104,7 +103,6 @@ impl Hash { // for v10 that use LE bytes. /// Hash data - #[cfg(feature = "v10")] pub fn data(data: &[u8]) -> Self { let (mut state, mut size) = State::new_with_size(8); state.encode_u64(data.len() as u64, &mut size); @@ -120,7 +118,6 @@ impl Hash { } /// Hash a parent - #[cfg(feature = "v10")] pub fn parent(left: &Node, right: &Node) -> Self { let (node1, node2) = if left.index <= right.index { (left, right) @@ -143,7 +140,6 @@ impl Hash { } /// Hash a tree - #[cfg(feature = "v10")] pub fn tree(roots: &[impl AsRef]) -> Self { let mut hasher = Blake2b::new(32); hasher.update(&ROOT_TYPE); @@ -187,7 +183,6 @@ impl DerefMut for Hash { /// Create a signable buffer for tree. This is treeSignable in Javascript. /// See https://github.com/hypercore-protocol/hypercore/blob/70b271643c4e4b1e5ecae5bb579966dfe6361ff3/lib/caps.js#L17 -#[cfg(feature = "v10")] pub fn signable_tree(hash: &[u8], length: u64, fork: u64) -> Box<[u8]> { let (mut state, mut buffer) = State::new_with_size(80); state.encode_fixed_32(&TREE, &mut buffer); @@ -285,7 +280,6 @@ mod tests { // https://github.com/mafintosh/hypercore-crypto/blob/master/test.js #[test] - #[cfg(feature = "v10")] fn hash_leaf() { let data = b"hello world"; check_hash( @@ -295,7 +289,6 @@ mod tests { } #[test] - #[cfg(feature = "v10")] fn hash_parent() { let data = b"hello world"; let len = data.len() as u64; @@ -308,7 +301,6 @@ mod tests { } #[test] - #[cfg(feature = "v10")] fn hash_tree() { let hash: [u8; 32] = [0; 32]; let node1 = Node::new(3, hash.to_vec(), 11); diff --git a/src/crypto/mod.rs b/src/crypto/mod.rs index 4cc10858..1909d2ea 100644 --- a/src/crypto/mod.rs +++ b/src/crypto/mod.rs @@ -4,7 +4,6 @@ mod hash; mod key_pair; mod merkle; -#[cfg(feature = "v10")] pub use self::hash::signable_tree; pub use self::hash::Hash; pub use self::key_pair::{ diff --git a/src/feed.rs b/src/feed.rs deleted file mode 100644 index 7b6f9e61..00000000 --- a/src/feed.rs +++ /dev/null @@ -1,677 +0,0 @@ -//! Hypercore's main abstraction. Exposes an append-only, secure log structure. - -pub use crate::common::Node; -use crate::feed_builder::FeedBuilder; -use crate::replicate::{Message, Peer}; -pub use crate::storage::{NodeTrait, Storage, Store}; - -use crate::audit::Audit; -use crate::bitfield::Bitfield; -use crate::crypto::{ - generate_keypair, sign, verify, Hash, Merkle, PublicKey, SecretKey, Signature, -}; -use crate::proof::Proof; -use anyhow::{bail, ensure, Result}; -use flat_tree as flat; -use pretty_hash::fmt as pretty_fmt; -use random_access_disk::RandomAccessDisk; -use random_access_memory::RandomAccessMemory; -use random_access_storage::RandomAccess; -use tree_index::TreeIndex; - -use std::borrow::Borrow; -use std::cmp; -use std::fmt::{self, Debug, Display}; -use std::ops::Range; -use std::path::Path; -use std::sync::Arc; - -/// Feed is an append-only log structure. -/// -/// To read an entry from a `Feed` you only need to know its [PublicKey], to write to a `Feed` -/// you must also have its [SecretKey]. The [SecretKey] should not be shared unless you know -/// what you're doing as only one client should be able to write to a single `Feed`. -/// If 2 seperate clients write conflicting information to the same `Feed` it will become corupted. -/// The feed needs an implementation of RandomAccess as a storage backing for the entrys added to it. -/// -/// There are several ways to construct a `Feed` -/// -/// __If you have a `Feed`'s [PublicKey], but have not opened a given `Feed` before__ -/// Use [builder] to initalize a new local `Feed` instance. You will not be able to write to this -/// feed. -/// -/// __If you want to create a new `Feed`__ -/// Use [with_storage] and `Feed` will create a new [SecretKey]/[PublicKey] keypair and store it -/// in the [Storage] -/// -/// __If you want to reopen a `Feed` you have previously opened__ -/// Use [with_storage], giving it a [Storage] that contains the previously opened `Feed` -/// -/// these references can be changed to the +nightly version, as docs.rs uses +nightly -/// -/// [SecretKey]: ed25519_dalek::SecretKey -/// [PublicKey]: ed25519_dalek::PublicKey -/// [RandomAccess]: random_access_storage::RandomAccess -/// [Storage]: crate::storage::Storage -/// [builder]: crate::feed_builder::FeedBuilder -/// [with_storage]: crate::feed::Feed::with_storage -#[derive(Debug)] -pub struct Feed -where - T: RandomAccess> + Debug, -{ - /// Merkle tree instance. - pub(crate) merkle: Merkle, - pub(crate) public_key: PublicKey, - pub(crate) secret_key: Option, - pub(crate) storage: Storage, - /// Total length of raw data stored in bytes. - pub(crate) byte_length: u64, - /// Total number of entries stored in the `Feed` - pub(crate) length: u64, - /// Bitfield to keep track of which data we own. - pub(crate) bitfield: Bitfield, - pub(crate) tree: TreeIndex, - pub(crate) peers: Vec, -} - -impl Feed -where - T: RandomAccess> + Debug + Send, -{ - /// Create a new instance with a custom storage backend. - pub async fn with_storage(mut storage: crate::storage::Storage) -> Result { - match storage.read_partial_keypair().await { - Some(partial_keypair) => { - let builder = FeedBuilder::new(partial_keypair.public, storage); - - // return early without secret key - if partial_keypair.secret.is_none() { - return Ok(builder.build().await?); - } - - builder - .secret_key( - partial_keypair - .secret - .ok_or_else(|| anyhow::anyhow!("secret-key not present"))?, - ) - .build() - .await - } - None => { - // we have no keys, generate a pair and save them to the storage - let keypair = generate_keypair(); - storage.write_public_key(&keypair.public).await?; - storage.write_secret_key(&keypair.secret).await?; - - FeedBuilder::new(keypair.public, storage) - .secret_key(keypair.secret) - .build() - .await - } - } - } - - /// Starts a `FeedBuilder` with the provided `PublicKey` and `Storage`. - pub fn builder(public_key: PublicKey, storage: Storage) -> FeedBuilder { - FeedBuilder::new(public_key, storage) - } - - /// Get the number of entries in the feed. - #[inline] - pub fn len(&self) -> u64 { - self.length - } - - /// Check if the length is 0. - #[inline] - pub fn is_empty(&self) -> bool { - self.len() == 0 - } - - /// Get the total amount of bytes stored in the feed. - #[inline] - pub fn byte_len(&self) -> u64 { - self.byte_length - } - - /// Append data into the log. /// - /// `append` will return an Err if this feed was not initalized with a [SecretKey]. - /// - /// It inserts the inputed data, it's signature, and a new [Merkle] node into [Storage]. - /// - /// [SecretKey]: ed25519_dalek::SecretKey - /// [Merkle]: crate::crypto::Merkle - /// [Storage]: crate::storage::Storage - #[inline] - pub async fn append(&mut self, data: &[u8]) -> Result<()> { - let key = match &self.secret_key { - Some(key) => key, - None => bail!("no secret key, cannot append."), - }; - self.merkle.next(data); - - self.storage - .write_data(self.byte_length as u64, &data) - .await?; - - let hash = Hash::from_roots(self.merkle.roots()); - let index = self.length; - let message = hash_with_length_as_bytes(hash, index + 1); - let signature = sign(&self.public_key, key, &message); - self.storage.put_signature(index, signature).await?; - - for node in self.merkle.nodes() { - self.storage.put_node(node).await?; - } - - self.byte_length += data.len() as u64; - - self.bitfield.set(index, true); - self.tree.set(tree_index(index)); - self.length += 1; - let bytes = self.bitfield.to_bytes(&self.tree)?; - self.storage.put_bitfield(0, &bytes).await?; - - Ok(()) - } - - /// Get the block of data at the tip of the feed. This will be the most - /// recently appended block. - #[inline] - pub async fn head(&mut self) -> Result>> { - match self.len() { - 0 => Ok(None), - len => self.get(len - 1).await, - } - } - - /// Return `true` if a data block is available locally. - #[inline] - pub fn has(&mut self, index: u64) -> bool { - self.bitfield.get(index) - } - - /// Return `true` if all data blocks within a range are available locally. - #[inline] - pub fn has_all(&mut self, range: ::std::ops::Range) -> bool { - let total = range.clone().count(); - total == self.bitfield.total_with_range(range) as usize - } - - /// Get the total amount of chunks downloaded. - #[inline] - pub fn downloaded(&mut self, range: ::std::ops::Range) -> u8 { - self.bitfield.total_with_range(range) - } - - /// Retrieve data from the log. - #[inline] - pub async fn get(&mut self, index: u64) -> Result>> { - if !self.bitfield.get(index) { - // NOTE: Do (network) lookup here once we have network code. - return Ok(None); - } - Ok(Some(self.storage.get_data(index).await?)) - } - - /// Return the Nodes which prove the correctness for the Node at index. - #[inline] - pub async fn proof(&mut self, index: u64, include_hash: bool) -> Result { - self.proof_with_digest(index, 0, include_hash).await - } - - /// Return the Nodes which prove the correctness for the Node at index with a - /// digest. - pub async fn proof_with_digest( - &mut self, - index: u64, - digest: u64, - include_hash: bool, - ) -> Result { - let mut remote_tree = TreeIndex::default(); - let mut nodes = vec![]; - - let proof = self.tree.proof_with_digest( - tree_index(index), - digest, - include_hash, - &mut nodes, - &mut remote_tree, - ); - - let proof = match proof { - Some(proof) => proof, - None => bail!("No proof available for index {}", index), - }; - - let tmp_num = proof.verified_by() / 2; - let (sig_index, has_underflow) = tmp_num.overflowing_sub(1); - let signature = if has_underflow { - None - } else { - match self.storage.get_signature(sig_index).await { - Ok(sig) => Some(sig), - Err(_) => None, - } - }; - - let mut nodes = Vec::with_capacity(proof.nodes().len()); - for index in proof.nodes() { - let node = self.storage.get_node(*index).await?; - nodes.push(node); - } - - Ok(Proof { - nodes, - signature, - index, - }) - } - - /// Compute the digest for the index. - pub fn digest(&mut self, index: u64) -> u64 { - self.tree.digest(tree_index(index)) - } - - /// Insert data into the tree at `index`. Verifies the `proof` when inserting - /// to make sure data is correct. Useful when replicating data from a remote - /// host. - pub async fn put(&mut self, index: u64, data: Option<&[u8]>, mut proof: Proof) -> Result<()> { - let mut next = tree_index(index); - let mut trusted: Option = None; - let mut missing = vec![]; - - let mut i = match data { - Some(_) => 0, - None => 1, - }; - - loop { - if self.tree.get(next) { - trusted = Some(next); - break; - } - let sibling = flat::sibling(next); - next = flat::parent(next); - if i < proof.nodes.len() && proof.nodes[i].index == sibling { - i += 1; - continue; - } - if !self.tree.get(sibling) { - break; - } - missing.push(sibling); - } - - if trusted.is_none() && self.tree.get(next) { - trusted = Some(next); - } - - let mut missing_nodes = vec![]; - for index in missing { - let node = self.storage.get_node(index).await?; - missing_nodes.push(node); - } - - let mut trusted_node = None; - if let Some(index) = trusted { - let node = self.storage.get_node(index).await?; - trusted_node = Some(node); - } - - let mut visited = vec![]; - let mut top = match data { - Some(data) => Node::new( - tree_index(index), - Hash::from_leaf(&data).as_bytes().to_owned(), - data.len() as u64, - ), - None => proof.nodes.remove(0), - }; - - // check if we already have the hash for this node - if verify_node(&trusted_node, &top) { - self.write(index, data, &visited, None).await?; - return Ok(()); - } - - // keep hashing with siblings until we reach the end or trusted node - loop { - let node; - let next = flat::sibling(top.index); - - if !proof.nodes.is_empty() && proof.nodes[0].index == next { - node = proof.nodes.remove(0); - visited.push(node.clone()); - } else if !missing_nodes.is_empty() && missing_nodes[0].index == next { - node = missing_nodes.remove(0); - } else { - // TODO: panics here - let nodes = self.verify_roots(&top, &mut proof).await?; - visited.extend_from_slice(&nodes); - self.write(index, data, &visited, proof.signature).await?; - return Ok(()); - } - - visited.push(top.clone()); - let hash = Hash::from_hashes(&top, &node); - let len = top.len() + node.len(); - top = Node::new(flat::parent(top.index), hash.as_bytes().into(), len); - - if verify_node(&trusted_node, &top) { - self.write(index, data, &visited, None).await?; - return Ok(()); - } - } - - fn verify_node(trusted: &Option, node: &Node) -> bool { - match trusted { - None => false, - Some(trusted) => trusted.index == node.index && trusted.hash == node.hash, - } - } - } - - /// Write some data to disk. Usually used in combination with `.put()`. - // in JS this calls to: - // - ._write() - // - ._onwrite() (emit the 'write' event), if it exists - // - ._writeAfterHook() (optionally going through writeHookdone()) - // - ._writeDone() - // - // Arguments are: (index, data, node, sig, from, cb) - async fn write( - &mut self, - index: u64, - data: Option<&[u8]>, - nodes: &[Node], - sig: Option, - ) -> Result<()> { - for node in nodes { - self.storage.put_node(node).await?; - } - - if let Some(data) = data { - self.storage.put_data(index, data, &nodes).await?; - } - - if let Some(sig) = sig { - let sig = sig.borrow(); - self.storage.put_signature(index, sig).await?; - } - - for node in nodes { - self.tree.set(node.index); - } - - self.tree.set(tree_index(index)); - - if let Some(_data) = data { - if self.bitfield.set(index, true).is_changed() { - // TODO: emit "download" event - } - // TODO: check peers.length, call ._announce if peers exist. - } - - // TODO: Discern between "primary" and "replica" streams. - // if (!this.writable) { - // if (!this._synced) this._synced = this.bitfield.iterator(0, this.length) - // if (this._synced.next() === -1) { - // this._synced.range(0, this.length) - // this._synced.seek(0) - // if (this._synced.next() === -1) { - // this.emit('sync') - // } - // } - // } - - Ok(()) - } - - /// Get a signature from the store. - pub async fn signature(&mut self, index: u64) -> Result { - ensure!( - index < self.length, - format!("No signature found for index {}", index) - ); - self.storage.next_signature(index).await - } - - /// Verify the entire feed. Checks a signature against the signature of all - /// root nodes combined. - pub async fn verify(&mut self, index: u64, signature: &Signature) -> Result<()> { - let roots = self.root_hashes(index).await?; - let roots: Vec<_> = roots.into_iter().map(Arc::new).collect(); - - let hash = Hash::from_roots(&roots); - let message = hash_with_length_as_bytes(hash, index + 1); - - verify_compat(&self.public_key, &message, Some(signature))?; - Ok(()) - } - - /// Announce we have a piece of data to all other peers. - // TODO: probably shouldn't be public - pub fn announce(&mut self, message: &Message, from: &Peer) { - for peer in &mut self.peers { - if peer != from { - peer.have(message) - } - } - } - - /// Announce we no longer have a piece of data to all other peers. - // TODO: probably shouldn't be public - pub fn unannounce(&mut self, message: &Message) { - for peer in &mut self.peers { - peer.unhave(message) - } - } - - /// Get all root hashes from the feed. - // In the JavaScript implementation this calls to `._getRootsToVerify()` - // internally. In Rust it seems better to just inline the code. - pub async fn root_hashes(&mut self, index: u64) -> Result> { - ensure!( - index <= self.length, - format!("Root index bounds exceeded {} > {}", index, self.length) - ); - let roots_index = tree_index(index) + 2; - let mut indexes = vec![]; - flat::full_roots(roots_index, &mut indexes); - - let mut roots = Vec::with_capacity(indexes.len()); - for index in indexes { - let node = self.storage.get_node(index).await?; - roots.push(node); - } - - Ok(roots) - } - - /// Access the public key. - pub fn public_key(&self) -> &PublicKey { - &self.public_key - } - - /// Access the secret key. - pub fn secret_key(&self) -> &Option { - &self.secret_key - } - - async fn verify_roots(&mut self, top: &Node, proof: &mut Proof) -> Result> { - let last_node = if !proof.nodes.is_empty() { - proof.nodes[proof.nodes.len() - 1].index - } else { - top.index - }; - - let verified_by = cmp::max(flat::right_span(top.index), flat::right_span(last_node)) + 2; - - let mut indexes = vec![]; - flat::full_roots(verified_by, &mut indexes); - let mut roots = Vec::with_capacity(indexes.len()); - let mut extra_nodes = vec![]; - - for index in indexes { - if index == top.index { - extra_nodes.push(top.clone()); - roots.push(top.clone()); // TODO: verify this is the right index to push to. - } else if !proof.nodes.is_empty() && index == proof.nodes[0].index { - extra_nodes.push(proof.nodes[0].clone()); - roots.push(proof.nodes.remove(0)); // TODO: verify this is the right index to push to. - } else if self.tree.get(index) { - let node = self.storage.get_node(index).await?; - roots.push(node); - } else { - bail!(": Missing tree roots needed for verify"); - } - } - - let checksum = Hash::from_roots(&roots); - let length = verified_by / 2; - let message = hash_with_length_as_bytes(checksum, length); - verify_compat(&self.public_key, &message, proof.signature())?; - - // Update the length if we grew the feed. - let len = verified_by / 2; - if len > self.len() { - self.length = len; - self.byte_length = roots.iter().fold(0, |acc, root| acc + root.index) - // TODO: emit('append') - } - - Ok(extra_nodes) - } - - /// Audit all data in the feed. Checks that all current data matches - /// the hashes in the merkle tree, and clears the bitfield if not. - /// The tuple returns is (valid_blocks, invalid_blocks) - pub async fn audit(&mut self) -> Result { - let mut valid_blocks = 0; - let mut invalid_blocks = 0; - for index in 0..self.length { - if self.bitfield.get(index) { - let node = self.storage.get_node(2 * index).await?; - let data = self.storage.get_data(index).await?; - let data_hash = Hash::from_leaf(&data); - if node.hash == data_hash.as_bytes() { - valid_blocks += 1; - } else { - invalid_blocks += 1; - self.bitfield.set(index, false); - } - } - } - Ok(Audit { - valid_blocks, - invalid_blocks, - }) - } - - /// Expose the bitfield attribute to use on during download - pub fn bitfield(&self) -> &Bitfield { - &self.bitfield - } - - /// (unimplemented) Provide a range of data to download. - pub fn download(&mut self, _range: Range) -> Result<()> { - unimplemented!(); - } - - /// (unimplemented) Provide a range of data to remove from the local storage. - pub fn undownload(&mut self, _range: Range) -> Result<()> { - unimplemented!(); - } - - /// (unimplemented) End the feed. - pub fn finalize(&mut self) -> Result<()> { - // if (!this.key) { - // this.key = crypto.tree(this._merkle.roots) - // this.discoveryKey = crypto.discoveryKey(this.key) - // } - // this._storage.key.write(0, this.key, cb) - unimplemented!(); - } - - /// Update all peers. - pub fn update_peers(&mut self) { - for peer in &mut self.peers { - peer.update(); - } - } -} - -impl Feed { - /// Create a new instance that persists to disk at the location of `dir`. - /// If dir was not there, it will be created. - // NOTE: Should we call these `data.bitfield` / `data.tree`? - pub async fn open>(path: P) -> Result { - if let Err(e) = std::fs::create_dir_all(&path) { - return Err(anyhow::Error::msg(format!( - "Failed to create directory {} because of: {}", - path.as_ref().display(), - e - ))); - } - - let dir = path.as_ref().to_owned(); - - let storage = Storage::new_disk(&dir, false).await?; - Self::with_storage(storage).await - } -} - -/// Create a new instance with an in-memory storage backend. -/// -/// ## Panics -/// Can panic if constructing the in-memory store fails, which is highly -/// unlikely. -impl Default for Feed { - fn default() -> Self { - async_std::task::block_on(async { - let storage = Storage::new_memory().await.unwrap(); - Self::with_storage(storage).await.unwrap() - }) - } -} - -impl> + Debug + Send> Display - for Feed -{ - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - // TODO: yay, we should find a way to convert this .unwrap() to an error - // type that's accepted by `fmt::Result<(), fmt::Error>`. - let key = pretty_fmt(&self.public_key.to_bytes()).unwrap(); - let byte_len = self.byte_len(); - let len = self.len(); - let peers = 0; // TODO: update once we actually have peers. - write!( - f, - "Hypercore(key=[{}], length={}, byte_length={}, peers={})", - key, len, byte_len, peers - ) - } -} - -/// Convert the index to the index in the tree. -#[inline] -fn tree_index(index: u64) -> u64 { - 2 * index -} - -/// Extend a hash with a big-endian encoded length. -fn hash_with_length_as_bytes(hash: Hash, length: u64) -> Vec { - [hash.as_bytes(), &length.to_be_bytes()].concat().to_vec() -} - -/// Verify a signature. If it fails, remove the length suffix added in Hypercore v9 -/// and verify again (backwards compatibility, remove in later version). -pub fn verify_compat(public: &PublicKey, msg: &[u8], sig: Option<&Signature>) -> Result<()> { - match verify(public, msg, sig) { - Ok(_) => Ok(()), - Err(_) => verify(public, &msg[0..32], sig), - } -} diff --git a/src/feed_builder.rs b/src/feed_builder.rs deleted file mode 100644 index 05d110fb..00000000 --- a/src/feed_builder.rs +++ /dev/null @@ -1,90 +0,0 @@ -use ed25519_dalek::{PublicKey, SecretKey}; - -use crate::bitfield::Bitfield; -use crate::crypto::Merkle; -use crate::storage::Storage; -use random_access_storage::RandomAccess; -use std::fmt::Debug; -use tree_index::TreeIndex; - -use crate::Feed; -use anyhow::Result; - -/// Construct a new `Feed` instance. -// TODO: make this an actual builder pattern. -// https://deterministic.space/elegant-apis-in-rust.html#builder-pattern -#[derive(Debug)] -pub struct FeedBuilder -where - T: RandomAccess + Debug, -{ - storage: Storage, - public_key: PublicKey, - secret_key: Option, -} - -impl FeedBuilder -where - T: RandomAccess> + Debug + Send, -{ - /// Create a new instance. - #[inline] - pub fn new(public_key: PublicKey, storage: Storage) -> Self { - Self { - storage, - public_key, - secret_key: None, - } - } - - /// Set the secret key. - pub fn secret_key(mut self, secret_key: SecretKey) -> Self { - self.secret_key = Some(secret_key); - self - } - - /// Finalize the builder. - #[inline] - pub async fn build(mut self) -> Result> { - let (bitfield, tree) = if let Ok(bitfield) = self.storage.read_bitfield().await { - Bitfield::from_slice(&bitfield) - } else { - Bitfield::new() - }; - - use crate::common::Node; - - let mut tree = TreeIndex::new(tree); - let mut roots = vec![]; - flat_tree::full_roots(tree.blocks() * 2, &mut roots); - let mut result: Vec> = vec![None; roots.len()]; - - for i in 0..roots.len() { - let node = self.storage.get_node(roots[i] as u64).await?; - let idx = roots - .iter() - .position(|&x| x == node.index) - .ok_or_else(|| anyhow::anyhow!("Couldnt find idx of node"))?; - result[idx] = Some(node); - } - - let roots = result - .into_iter() - .collect::>>() - .ok_or_else(|| anyhow::anyhow!("Roots contains undefined nodes"))?; - - let byte_length = roots.iter().fold(0, |acc, node| acc + node.length); - - Ok(Feed { - merkle: Merkle::from_nodes(roots), - byte_length, - length: tree.blocks(), - bitfield, - tree, - public_key: self.public_key, - secret_key: self.secret_key, - storage: self.storage, - peers: vec![], - }) - } -} diff --git a/src/lib.rs b/src/lib.rs index a7650bd2..9b452cdb 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -13,20 +13,6 @@ //! //! ## Example //! ```rust -//! #[cfg(feature = "v9")] -//! # fn main() -> Result<(), Box> { -//! # async_std::task::block_on(async { -//! let mut feed = hypercore::open("./feed.db").await?; -//! -//! feed.append(b"hello").await?; -//! feed.append(b"world").await?; -//! -//! assert_eq!(feed.get(0).await?, Some(b"hello".to_vec())); -//! assert_eq!(feed.get(1).await?, Some(b"world".to_vec())); -//! # Ok(()) -//! # }) -//! # } -//! #[cfg(feature = "v10")] //! # fn main() -> Result<(), Box> { //! # async_std::task::block_on(async { //! // unimplemented @@ -39,70 +25,29 @@ //! [Dat]: https://github.com/datrs //! [Feed]: crate::feed::Feed -#[cfg(feature = "v9")] -pub mod bitfield; -#[cfg(feature = "v10")] pub mod compact_encoding; pub mod prelude; -mod audit; - -#[cfg(feature = "v10")] mod bitfield_v10; mod common; -#[cfg(feature = "v10")] mod core; mod crypto; -#[cfg(feature = "v10")] mod data; mod event; -#[cfg(feature = "v9")] -mod feed; -#[cfg(feature = "v9")] -mod feed_builder; -#[cfg(feature = "v10")] mod oplog; -mod proof; -mod replicate; -#[cfg(feature = "v9")] -mod storage; -#[cfg(feature = "v10")] mod storage_v10; -#[cfg(feature = "v10")] mod tree; pub use crate::common::Node; -#[cfg(feature = "v10")] pub use crate::common::{ DataBlock, DataHash, DataSeek, DataUpgrade, Proof, RequestBlock, RequestSeek, RequestUpgrade, Store, }; -#[cfg(feature = "v10")] pub use crate::core::Hypercore; pub use crate::crypto::{generate_keypair, sign, verify, Signature}; pub use crate::event::Event; -#[cfg(feature = "v9")] -pub use crate::feed::Feed; -#[cfg(feature = "v9")] -pub use crate::feed_builder::FeedBuilder; -#[cfg(feature = "v9")] -pub use crate::proof::Proof; -pub use crate::replicate::Peer; -#[cfg(feature = "v9")] -pub use crate::storage::{NodeTrait, PartialKeypair, Storage, Store}; -#[cfg(feature = "v10")] pub use crate::storage_v10::{PartialKeypair, Storage}; pub use ed25519_dalek::{ ExpandedSecretKey, Keypair, PublicKey, SecretKey, EXPANDED_SECRET_KEY_LENGTH, KEYPAIR_LENGTH, PUBLIC_KEY_LENGTH, SECRET_KEY_LENGTH, }; - -use std::path::Path; - -/// Create a new Hypercore `Feed`. -#[cfg(feature = "v9")] -pub async fn open>( - path: P, -) -> anyhow::Result> { - Feed::open(path).await -} diff --git a/src/prelude.rs b/src/prelude.rs index 5c2c1980..f6bc2097 100644 --- a/src/prelude.rs +++ b/src/prelude.rs @@ -1,19 +1,4 @@ //! Convenience wrapper to import all of Hypercore's core. -//! -//! ```rust -//! use hypercore::prelude::*; -//! #[cfg(feature = "v9")] -//! let feed = Feed::default(); -//! ``` -#[cfg(feature = "v9")] -pub use crate::feed::Feed; -// pub use feed_builder::FeedBuilder; -pub use crate::common::Node; -#[cfg(feature = "v10")] pub use crate::common::Store; -#[cfg(feature = "v10")] pub use crate::core::Hypercore; -#[cfg(feature = "v9")] -pub use crate::storage::{NodeTrait, Storage, Store}; -#[cfg(feature = "v10")] pub use crate::storage_v10::{PartialKeypair, Storage}; diff --git a/src/proof.rs b/src/proof.rs deleted file mode 100644 index e7d4cd42..00000000 --- a/src/proof.rs +++ /dev/null @@ -1,30 +0,0 @@ -use crate::Node; -use crate::Signature; - -/// A merkle proof for an index, created by the `.proof()` method. -#[derive(Debug, PartialEq, Clone)] -pub struct Proof { - /// The index to which this proof corresponds. - pub index: u64, - /// Nodes that verify the index you passed. - pub nodes: Vec, - /// An `ed25519` signature, guaranteeing the integrity of the nodes. - pub signature: Option, -} - -impl Proof { - /// Access the `index` field from the proof. - pub fn index(&self) -> u64 { - self.index - } - - /// Access the `nodes` field from the proof. - pub fn nodes(&self) -> &[Node] { - &self.nodes - } - - /// Access the `signature` field from the proof. - pub fn signature(&self) -> Option<&Signature> { - self.signature.as_ref() - } -} diff --git a/src/replicate/message.rs b/src/replicate/message.rs deleted file mode 100644 index 8a8887e9..00000000 --- a/src/replicate/message.rs +++ /dev/null @@ -1,6 +0,0 @@ -/// A message sent over the network. -#[derive(Debug, Clone, PartialEq)] -pub struct Message { - start: u64, - length: Option, -} diff --git a/src/replicate/mod.rs b/src/replicate/mod.rs deleted file mode 100644 index d0f44e5c..00000000 --- a/src/replicate/mod.rs +++ /dev/null @@ -1,5 +0,0 @@ -mod message; -mod peer; - -pub use self::message::Message; -pub use self::peer::Peer; diff --git a/src/replicate/peer.rs b/src/replicate/peer.rs deleted file mode 100644 index 58e9357b..00000000 --- a/src/replicate/peer.rs +++ /dev/null @@ -1,40 +0,0 @@ -// use sparse_bitfield::Bitfield; - -use super::Message; - -/// A peer on the network. -// omitted fields: [ -// feed, -// stream, -// inflightRequests, -// ] -#[derive(Debug, Clone, PartialEq)] -pub struct Peer { - // remote_id: u64, -// remote_length: u64, -// remote_bitfield: Bitfield, -// remote_is_want: bool, -// remote_is_downloading: bool, -// is_live: bool, -// is_sparse: bool, -// is_downloading: bool, -// is_uploading: bool, -// max_requests: u16, -} - -impl Peer { - /// Check if the peer has a message. - pub fn have(&mut self, _msg: &Message) { - unimplemented!(); - } - - /// Tell a peer you no longer have a message. - pub fn unhave(&mut self, _msg: &Message) { - unimplemented!(); - } - - /// Update. - pub fn update(&mut self) { - unimplemented!(); - } -} diff --git a/src/storage/mod.rs b/src/storage/mod.rs deleted file mode 100644 index ff869518..00000000 --- a/src/storage/mod.rs +++ /dev/null @@ -1,444 +0,0 @@ -//! Save data to a desired storage backend. - -mod persist; - -pub use merkle_tree_stream::Node as NodeTrait; -pub use self::persist::Persist; - -use crate::common::Node; -use anyhow::{anyhow, ensure, Result}; -use ed25519_dalek::{PublicKey, SecretKey, Signature, PUBLIC_KEY_LENGTH, SECRET_KEY_LENGTH}; -use flat_tree as flat; -use futures::future::FutureExt; -#[cfg(not(target_arch = "wasm32"))] -use random_access_disk::RandomAccessDisk; -use random_access_memory::RandomAccessMemory; -use random_access_storage::RandomAccess; -use sleep_parser::{create_bitfield, create_signatures, create_tree, Header}; -use std::borrow::Borrow; -use std::convert::TryFrom; -use std::fmt::Debug; -use std::ops::Range; -use std::path::PathBuf; - -const HEADER_OFFSET: u64 = 32; - -/// Key pair where for read-only hypercores the secret key can also be missing. -#[derive(Debug)] -pub struct PartialKeypair { - /// Public key - pub public: PublicKey, - /// Secret key. If None, the hypercore is read-only. - pub secret: Option, -} - -/// The types of stores that can be created. -#[derive(Debug)] -pub enum Store { - /// Tree - Tree, - /// Data - Data, - /// Bitfield - Bitfield, - /// Signatures - Signatures, - /// Keypair - Keypair, -} - -/// Save data to a desired storage backend. -#[derive(Debug)] -pub struct Storage -where - T: RandomAccess + Debug, -{ - tree: T, - data: T, - bitfield: T, - signatures: T, - keypair: T, -} - -impl Storage -where - T: RandomAccess> + Debug + Send, -{ - /// Create a new instance. Takes a keypair and a callback to create new - /// storage instances. - // Named `.open()` in the JS version. Replaces the `.openKey()` method too by - // requiring a key pair to be initialized before creating a new instance. - pub async fn new(create: Cb, overwrite: bool) -> Result - where - Cb: Fn(Store) -> std::pin::Pin> + Send>>, - { - let tree = create(Store::Tree).await?; - let data = create(Store::Data).await?; - let bitfield = create(Store::Bitfield).await?; - let signatures = create(Store::Signatures).await?; - let keypair = create(Store::Keypair).await?; - - let mut instance = Self { - tree, - data, - bitfield, - signatures, - keypair, - }; - - if overwrite || instance.bitfield.len().await.unwrap_or(0) == 0 { - let header = create_bitfield(); - instance - .bitfield - .write(0, &header.to_vec()) - .await - .map_err(|e| anyhow!(e))?; - } - - if overwrite || instance.signatures.len().await.unwrap_or(0) == 0 { - let header = create_signatures(); - instance - .signatures - .write(0, &header.to_vec()) - .await - .map_err(|e| anyhow!(e))?; - } - - if overwrite || instance.tree.len().await.unwrap_or(0) == 0 { - let header = create_tree(); - instance - .tree - .write(0, &header.to_vec()) - .await - .map_err(|e| anyhow!(e))?; - } - - Ok(instance) - } - - /// Write data to the feed. - #[inline] - pub async fn write_data(&mut self, offset: u64, data: &[u8]) -> Result<()> { - self.data.write(offset, &data).await.map_err(|e| anyhow!(e)) - } - - /// Write a byte vector to a data storage (random-access instance) at the - /// position of `index`. - /// - /// NOTE: Meant to be called from the `.put()` feed method. Probably used to - /// insert data as-is after receiving it from the network (need to confirm - /// with mafintosh). - /// TODO: Ensure the signature size is correct. - /// NOTE: Should we create a `Data` entry type? - pub async fn put_data(&mut self, index: u64, data: &[u8], nodes: &[Node]) -> Result<()> { - if data.is_empty() { - return Ok(()); - } - - let range = self.data_offset(index, nodes).await?; - - ensure!( - (range.end - range.start) as usize == data.len(), - format!("length `{:?} != {:?}`", range.count(), data.len()) - ); - - self.data - .write(range.start, data) - .await - .map_err(|e| anyhow!(e)) - } - - /// Get data from disk that the user has written to it. This is stored - /// unencrypted, so there's no decryption needed. - // FIXME: data_offset always reads out index 0, length 0 - #[inline] - pub async fn get_data(&mut self, index: u64) -> Result> { - let cached_nodes = Vec::new(); // TODO: reuse allocation. - let range = self.data_offset(index, &cached_nodes).await?; - self.data - .read(range.start, range.count() as u64) - .await - .map_err(|e| anyhow!(e)) - } - - /// Search the signature stores for a `Signature`, starting at `index`. - pub fn next_signature( - &mut self, - index: u64, - ) -> futures::future::BoxFuture<'_, Result> { - let bytes = async_std::task::block_on(async { - self.signatures - .read(HEADER_OFFSET + 64 * index, 64) - .await - .map_err(|e| anyhow!(e)) - }); - async move { - let bytes = bytes?; - if not_zeroes(&bytes) { - Ok(Signature::try_from(&bytes[..])?) - } else { - Ok(self.next_signature(index + 1).await?) - } - } - .boxed() - } - - /// Get a `Signature` from the store. - #[inline] - pub async fn get_signature(&mut self, index: u64) -> Result { - let bytes = self - .signatures - .read(HEADER_OFFSET + 64 * index, 64) - .await - .map_err(|e| anyhow!(e))?; - ensure!(not_zeroes(&bytes), "No signature found"); - Ok(Signature::try_from(&bytes[..])?) - } - - /// Write a `Signature` to `self.Signatures`. - /// TODO: Ensure the signature size is correct. - /// NOTE: Should we create a `Signature` entry type? - #[inline] - pub async fn put_signature( - &mut self, - index: u64, - signature: impl Borrow, - ) -> Result<()> { - let signature = signature.borrow(); - self.signatures - .write(HEADER_OFFSET + 64 * index, &signature.to_bytes()) - .await - .map_err(|e| anyhow!(e)) - } - - /// TODO(yw) docs - /// Get the offset for the data, return `(offset, size)`. - /// - /// ## Panics - /// A panic can occur if no maximum value is found. - pub async fn data_offset(&mut self, index: u64, cached_nodes: &[Node]) -> Result> { - let mut roots = Vec::new(); // TODO: reuse alloc - flat::full_roots(tree_index(index), &mut roots); - - let mut offset = 0; - let mut pending = roots.len() as u64; - let block_index = tree_index(index); - - if pending == 0 { - let len = match find_node(&cached_nodes, block_index) { - Some(node) => node.len(), - None => (self.get_node(block_index).await?).len(), - }; - return Ok(offset..offset + len); - } - - for root in roots { - // FIXME: we're always having a cache miss here. Check cache first before - // getting a node from the backend. - // - // ```rust - // let node = match find_node(cached_nodes, root) { - // Some(node) => node, - // None => self.get_node(root), - // }; - // ``` - let node = self.get_node(root).await?; - - offset += node.len(); - pending -= 1; - if pending > 0 { - continue; - } - - let len = match find_node(&cached_nodes, block_index) { - Some(node) => node.len(), - None => (self.get_node(block_index).await?).len(), - }; - - return Ok(offset..offset + len); - } - - unreachable!(); - } - - /// Get a `Node` from the `tree` storage. - #[inline] - pub async fn get_node(&mut self, index: u64) -> Result { - let buf = self - .tree - .read(HEADER_OFFSET + 40 * index, 40) - .await - .map_err(|e| anyhow!(e))?; - let node = Node::from_bytes(index, &buf)?; - Ok(node) - } - - /// Write a `Node` to the `tree` storage. - /// TODO: prevent extra allocs here. Implement a method on node that can reuse - /// a buffer. - #[inline] - pub async fn put_node(&mut self, node: &Node) -> Result<()> { - let index = node.index(); - let buf = node.to_bytes()?; - self.tree - .write(HEADER_OFFSET + 40 * index, &buf) - .await - .map_err(|e| anyhow!(e)) - } - - /// Write data to the internal bitfield module. - /// TODO: Ensure the chunk size is correct. - /// NOTE: Should we create a bitfield entry type? - #[inline] - pub async fn put_bitfield(&mut self, offset: u64, data: &[u8]) -> Result<()> { - self.bitfield - .write(HEADER_OFFSET + offset, data) - .await - .map_err(|e| anyhow!(e)) - } - - /// Read bitfield header. - pub async fn read_bitfield(&mut self) -> Result> { - let buf = self - .bitfield - .read(0, 32) - .await - .map_err(|_| anyhow::anyhow!("read bitfield header"))?; - let header = Header::from_vec(&buf).map_err(|e| anyhow::anyhow!(e))?; - - // khodzha: - // TODO: we should handle eof vs errors here somehow but idk how to do that - let mut buf: Vec = Vec::new(); - let mut idx: u64 = 0; - let ent_size: u64 = header.entry_size as u64; - loop { - let result = self - .bitfield - .read(HEADER_OFFSET + idx * ent_size, ent_size) - .await; - if let Ok(slice) = result { - buf.extend_from_slice(&slice); - idx += 1; - } else { - return Ok(buf); - } - } - } - - /// Read a public key from storage - pub async fn read_public_key(&mut self) -> Result { - let buf = self - .keypair - .read(0, PUBLIC_KEY_LENGTH as u64) - .await - .map_err(|e| anyhow!(e))?; - let public_key = PublicKey::from_bytes(&buf)?; - Ok(public_key) - } - - /// Read a secret key from storage - pub async fn read_secret_key(&mut self) -> Result { - let buf = self - .keypair - .read(PUBLIC_KEY_LENGTH as u64, SECRET_KEY_LENGTH as u64) - .await - .map_err(|e| anyhow!(e))?; - let secret_key = SecretKey::from_bytes(&buf)?; - Ok(secret_key) - } - - /// Write a public key to the storage - pub async fn write_public_key(&mut self, public_key: &PublicKey) -> Result<()> { - let buf: [u8; PUBLIC_KEY_LENGTH] = public_key.to_bytes(); - self.keypair.write(0, &buf).await.map_err(|e| anyhow!(e)) - } - - /// Write a secret key to the storage - pub async fn write_secret_key(&mut self, secret_key: &SecretKey) -> Result<()> { - let buf: [u8; SECRET_KEY_LENGTH] = secret_key.to_bytes(); - self.keypair - .write(PUBLIC_KEY_LENGTH as u64, &buf) - .await - .map_err(|e| anyhow!(e)) - } - - /// Tries to read a partial keypair (ie: with an optional secret_key) from the storage - pub async fn read_partial_keypair(&mut self) -> Option { - match self.read_public_key().await { - Ok(public) => match self.read_secret_key().await { - Ok(secret) => Some(PartialKeypair { - public, - secret: Some(secret), - }), - Err(_) => Some(PartialKeypair { - public, - secret: None, - }), - }, - Err(_) => None, - } - } -} - -impl Storage { - /// Create a new instance backed by a `RandomAccessMemory` instance. - pub async fn new_memory() -> Result { - let create = |_| async { Ok(RandomAccessMemory::default()) }.boxed(); - Ok(Self::new(create, true).await?) - } -} - -#[cfg(not(target_arch = "wasm32"))] -impl Storage { - /// Create a new instance backed by a `RandomAccessDisk` instance. - pub async fn new_disk(dir: &PathBuf, overwrite: bool) -> Result { - let storage = |storage: Store| { - let name = match storage { - Store::Tree => "tree", - Store::Data => "data", - Store::Bitfield => "bitfield", - Store::Signatures => "signatures", - Store::Keypair => "key", - }; - RandomAccessDisk::open(dir.as_path().join(name)).boxed() - }; - Ok(Self::new(storage, overwrite).await?) - } -} - -/// Get a node from a vector of nodes. -#[inline] -fn find_node(nodes: &[Node], index: u64) -> Option<&Node> { - for node in nodes { - if node.index() == index { - return Some(node); - } - } - None -} - -/// Check if a byte slice is not completely zero-filled. -#[inline] -fn not_zeroes(bytes: &[u8]) -> bool { - for byte in bytes { - if *byte != 0 { - return true; - } - } - false -} - -/// Convert the index to the index in the tree. -#[inline] -fn tree_index(index: u64) -> u64 { - 2 * index -} - -#[test] -fn should_detect_zeroes() { - let nums = vec![0; 10]; - assert!(!not_zeroes(&nums)); - - let nums = vec![1; 10]; - assert!(not_zeroes(&nums)); -} diff --git a/src/storage/persist.rs b/src/storage/persist.rs deleted file mode 100644 index 70a3ec0a..00000000 --- a/src/storage/persist.rs +++ /dev/null @@ -1,19 +0,0 @@ -use super::Storage; -use anyhow::Result; -use random_access_storage::RandomAccess; -use std::fmt::Debug; - -/// Persist data to a `Storage` instance. -pub trait Persist -where - T: RandomAccess + Debug, -{ - /// Create an instance from a byte vector. - fn from_bytes(index: u64, buf: &[u8]) -> Self; - - /// Create a vector. - fn to_vec(&self) -> Result>; - - /// Persist into a storage backend. - fn store(&self, index: u64, store: Storage) -> Result<()>; -} diff --git a/tests/bitfield.rs b/tests/bitfield.rs deleted file mode 100644 index 4c4ea182..00000000 --- a/tests/bitfield.rs +++ /dev/null @@ -1,204 +0,0 @@ -use rand; - -#[cfg(feature = "v9")] -use hypercore::bitfield::{Bitfield, Change::*}; -#[cfg(feature = "v9")] -use rand::Rng; - -#[test] -#[cfg(feature = "v9")] -fn set_and_get() { - let (mut b, _) = Bitfield::new(); - - assert_eq!(b.get(0), false); - assert_eq!(b.set(0, true), Changed); - assert_eq!(b.set(0, true), Unchanged); - assert_eq!(b.get(0), true); - - assert_eq!(b.get(1_424_244), false); - assert_eq!(b.set(1_424_244, true), Changed); - assert_eq!(b.set(1_424_244, true), Unchanged); - assert_eq!(b.get(1_424_244), true); -} - -#[test] -#[cfg(feature = "v9")] -fn set_and_get_tree() { - let (mut b, mut tree) = Bitfield::new(); - - { - assert_eq!(tree.get(0), false); - assert_eq!(tree.set(0, true), Changed); - assert_eq!(tree.set(0, true), Unchanged); - assert_eq!(tree.get(0), true); - - assert_eq!(tree.get(1_424_244), false); - assert_eq!(tree.set(1_424_244, true), Changed); - assert_eq!(tree.set(1_424_244, true), Unchanged); - assert_eq!(tree.get(1_424_244), true); - } - - assert_eq!(b.get(0), false); - assert_eq!(b.get(1_424_244), false); -} - -#[test] -#[cfg(feature = "v9")] -fn set_and_index() { - let (mut b, _) = Bitfield::new(); - - { - let mut iter = b.iterator_with_range(0, 100_000_000); - assert_eq!(iter.next(), Some(0)); - } - - b.set(0, true); - { - let mut iter = b.iterator_with_range(0, 100_000_000); - assert_eq!(iter.seek(0).next(), Some(1)); - } - - b.set(479, true); - { - let mut iter = b.iterator_with_range(0, 100_000_000); - assert_eq!(iter.seek(478).next(), Some(478)); - assert_eq!(iter.next(), Some(480)); - } - - b.set(1, true); - { - let mut iter = b.iterator_with_range(0, 100_000_000); - assert_eq!(iter.seek(0).next(), Some(2)); - } - - b.set(2, true); - { - let mut iter = b.iterator_with_range(0, 100_000_000); - assert_eq!(iter.seek(0).next(), Some(3)); - } - - b.set(3, true); - { - let mut iter = b.iterator_with_range(0, 100_000_000); - assert_eq!(iter.seek(0).next(), Some(4)); - } - - let len = b.len(); - for i in 0..len { - b.set(i, true); - } - { - let mut iter = b.iterator_with_range(0, 100_000_000); - assert_eq!(iter.seek(0).next(), Some(len)); - } - - for i in 0..len { - b.set(i, false); - } - { - let mut iter = b.iterator_with_range(0, 100_000_000); - assert_eq!(iter.seek(0).next(), Some(0)); - } -} - -#[test] -#[cfg(feature = "v9")] -fn set_and_index_random() { - let (mut b, _) = Bitfield::new(); - - let mut rng = rand::thread_rng(); - for _ in 0..100 { - assert!(check(&mut b), "index validates"); - set(&mut b, rng.gen_range(0, 2000), rng.gen_range(0, 8)); - } - - assert!(check(&mut b), "index validates"); - - fn check(b: &mut Bitfield) -> bool { - let mut all = vec![true; b.len() as usize]; - - { - let mut iter = b.iterator(); - - while let Some(i) = iter.next() { - all[i as usize] = false; - } - } - - for (i, &v) in all.iter().enumerate() { - if b.get(i as u64) != v { - return false; - } - } - - true - } - - fn set(b: &mut Bitfield, i: u64, n: u64) { - for j in i..i + n { - b.set(j, true); - } - } -} - -#[test] -#[cfg(feature = "v9")] -fn get_total_positive_bits() { - let (mut b, _) = Bitfield::new(); - - assert_eq!(b.set(1, true), Changed); - assert_eq!(b.set(2, true), Changed); - assert_eq!(b.set(4, true), Changed); - assert_eq!(b.set(5, true), Changed); - assert_eq!(b.set(39, true), Changed); - - assert_eq!(b.total_with_range(0..4), 2); - assert_eq!(b.total_with_range(3..4), 0); - assert_eq!(b.total_with_range(3..5), 1); - assert_eq!(b.total_with_range(3..40), 3); - assert_eq!(b.total(), 5); - assert_eq!(b.total_with_start(7), 1); -} - -#[test] -#[cfg(feature = "v9")] -fn bitfield_dedup() { - let (mut b, mut tree) = Bitfield::new(); - - for i in 0..32 * 1024 { - b.set(i, true); - } - - for i in 0..64 * 1024 { - tree.set(i, true); - } - - assert!(b.get(8 * 1024)); - assert!(b.get(16 * 1024)); - b.set(8 * 1024, false); - assert!(!b.get(8 * 1024)); - assert!(b.get(16 * 1024)); -} - -#[test] -#[cfg(feature = "v9")] -fn bitfield_compress() { - let (mut b, _) = Bitfield::new(); - assert_eq!(b.compress(0, 0).unwrap(), vec![0]); - - b.set(1, true); - assert_eq!(b.compress(0, 0).unwrap(), vec![2, 64, 253, 31]); - - b.set(1_424_244, true); - assert_eq!( - b.compress(0, 0).unwrap(), - vec![2, 64, 181, 187, 43, 2, 8, 197, 4] - ); - assert_eq!(b.compress(0, 1).unwrap(), vec![2, 64, 253, 31]); - assert_eq!( - b.compress(1_424_244, 1).unwrap(), - vec![185, 27, 2, 8, 197, 4] - ); - - assert_eq!(b.compress(1_424_244_000, 1).unwrap(), vec![0]); -} diff --git a/tests/common/mod.rs b/tests/common/mod.rs index a470166c..065a82db 100644 --- a/tests/common/mod.rs +++ b/tests/common/mod.rs @@ -1,13 +1,8 @@ -use hypercore::PartialKeypair; - -use anyhow::Error; use ed25519_dalek::{PublicKey, SecretKey, PUBLIC_KEY_LENGTH, SECRET_KEY_LENGTH}; -use futures::future::FutureExt; -#[cfg(feature = "v9")] -use hypercore::{Feed, Storage, Store}; -use random_access_memory as ram; use sha2::{Digest, Sha256}; +use hypercore::PartialKeypair; + const TEST_PUBLIC_KEY_BYTES: [u8; PUBLIC_KEY_LENGTH] = [ 0x97, 0x60, 0x6c, 0xaa, 0xd2, 0xb0, 0x8c, 0x1d, 0x5f, 0xe1, 0x64, 0x2e, 0xee, 0xa5, 0x62, 0xcb, 0x91, 0xd6, 0x55, 0xe2, 0x00, 0xc8, 0xd4, 0x3a, 0x32, 0x09, 0x1d, 0x06, 0x4a, 0x33, 0x1e, 0xe3, @@ -20,13 +15,6 @@ const TEST_SECRET_KEY_BYTES: [u8; SECRET_KEY_LENGTH] = [ 0x51, 0x0b, 0x71, 0x14, 0x15, 0xf3, 0x31, 0xf6, 0xfa, 0x9e, 0xf2, 0xbf, 0x23, 0x5f, 0x2f, 0xfe, ]; -#[cfg(feature = "v9")] -pub async fn create_feed(page_size: usize) -> Result, Error> { - let create = |_store: Store| async move { Ok(ram::RandomAccessMemory::new(page_size)) }.boxed(); - let storage = Storage::new(create, false).await?; - Feed::with_storage(storage).await -} - #[derive(PartialEq, Debug)] pub struct HypercoreHash { pub bitfield: Option, diff --git a/tests/compact_encoding.rs b/tests/compact_encoding.rs index 7df59029..bcf10c15 100644 --- a/tests/compact_encoding.rs +++ b/tests/compact_encoding.rs @@ -1,5 +1,3 @@ -#![cfg(feature = "v10")] - use hypercore::compact_encoding::{CompactEncoding, State}; // The max value for 1 byte length is 252 diff --git a/tests/compat.rs b/tests/compat.rs deleted file mode 100644 index 1f8a2da4..00000000 --- a/tests/compat.rs +++ /dev/null @@ -1,188 +0,0 @@ -//! Based on https://github.com/mafintosh/hypercore/blob/cf08d8c907e302cf4b699738f229b050eba41b59/test/compat.js - -use ed25519_dalek; - -use tempfile; - -use std::convert::TryFrom; -use std::fs::File; -use std::io::Read; -use std::path::{Path, PathBuf}; - -use data_encoding::HEXLOWER; -use ed25519_dalek::{Keypair, Signature}; -#[cfg(feature = "v9")] -use hypercore::Feed; -use hypercore::{Storage, Store}; -use random_access_disk::RandomAccessDisk; -use remove_dir_all::remove_dir_all; - -#[async_std::test] -#[cfg(feature = "v9")] -async fn deterministic_data_and_tree() { - let expected_tree = hex_bytes(concat!( - "0502570200002807424c414b4532620000000000000000000000000000000000ab27d45f509274", - "ce0d08f4f09ba2d0e0d8df61a0c2a78932e81b5ef26ef398df0000000000000001064321a8413b", - "e8c604599689e2c7a59367b031b598bceeeb16556a8f3252e0de000000000000000294c1705400", - "5942a002c7c39fbb9c6183518691fb401436f1a2f329b380230af800000000000000018dfe81d5", - "76464773f848b9aba1c886fde57a49c283ab57f4a297d976d986651e00000000000000041d2fad", - "c9ce604c7e592949edc964e45aaa10990d7ee53328439ef9b2cf8aa6ff00000000000000013a8d", - "cc74e80b8314e8e13e1e462358cf58cf5fc4413a9b18a891ffacc551c395000000000000000228", - "28647a654a712738e35f49d1c05c676010be0b33882affc1d1e7e9fee59d400000000000000001", - "000000000000000000000000000000000000000000000000000000000000000000000000000000", - "00baac70b6d38243efa028ee977c462e4bec73d21d09ceb8cc16f4d4b1ee228a45000000000000", - "0001d1b021632c7fab84544053379112ca7b165bb21283821816c5b6c89ff7f78e2d0000000000", - "000002d2ab421cece792033058787a5ba72f3a701fddc25540d5924e9819d7c12e02f200000000", - "00000001" - )); - - for _ in 0..5 { - let (dir, storage) = mk_storage().await; - let mut feed = Feed::with_storage(storage).await.unwrap(); - - let data = b"abcdef"; - for &b in data { - feed.append(&[b]).await.unwrap(); - } - assert_eq!(read_bytes(&dir, Store::Data), data); - assert_eq!(read_bytes(&dir, Store::Tree), expected_tree); - - remove_dir_all(dir).unwrap() - } -} - -#[test] -#[ignore] -fn deterministic_data_and_tree_after_replication() { - // Port from mafintosh/hypercore when the necessary features are implemented - unimplemented!(); -} - -#[async_std::test] -#[cfg(feature = "v9")] -async fn deterministic_signatures() { - let key = hex_bytes("9718a1ff1c4ca79feac551c0c7212a65e4091278ec886b88be01ee4039682238"); - let keypair_bytes = hex_bytes(concat!( - "53729c0311846cca9cc0eded07aaf9e6689705b6a0b1bb8c3a2a839b72fda383", - "9718a1ff1c4ca79feac551c0c7212a65e4091278ec886b88be01ee4039682238" - )); - - let compat_v9_expected_signatures = hex_bytes(concat!( - "050257010000400745643235353139000000000000000000000000000000000084684e8dd76c339", - "d6f5754e813204906ee818e6c6cdc6a816a2ac785a3e0d926ac08641a904013194fe6121847b7da", - "d4e361965d47715428eb0a0ededbdd5909d037ff3c3614fa0100ed9264a712d3b77cbe7a4f6eadd", - "8f342809be99dfb9154a19e278d7a5de7d2b4d890f7701a38b006469f6bab1aff66ac6125d48baf", - "dc0711057675ed57d445ce7ed4613881be37ebc56bb40556b822e431bb4dc3517421f9a5e3ed124", - "eb5c4db8367386d9ce12b2408613b9fec2837022772a635ffd807", - )); - let compat_signatures_len = compat_v9_expected_signatures.len(); - let compat_signature_struct = compat_v9_expected_signatures - .into_iter() - .skip(compat_signatures_len - 64) - .collect::>(); - - let expected_signatures = hex_bytes(concat!( - "42e057f2c225b4c5b97876a15959324931ad84646a8bf2e4d14487c0f117966a585edcdda54670d", - "d5def829ca85924ce44ae307835e57d5729aef8cd91678b06", - )); - - for _ in 0..5 { - let (dir, storage) = mk_storage().await; - let keypair = mk_keypair(&keypair_bytes, &key); - let mut feed = Feed::builder(keypair.public, storage) - .secret_key(keypair.secret) - .build() - .await - .unwrap(); - - let data = b"abc"; - for &b in data { - feed.append(&[b]).await.unwrap(); - } - - assert_eq!(read_bytes(&dir, Store::Data), data); - let actual_signatures = read_bytes(&dir, Store::Signatures); - let actual_signatures_len = actual_signatures.len(); - assert_eq!( - actual_signatures - .into_iter() - .skip(actual_signatures_len - 64) - .collect::>(), - expected_signatures - ); - - let compat_signature = Signature::try_from(&compat_signature_struct[..]).unwrap(); - feed.verify(feed.len() - 1, &compat_signature) - .await - .expect("Could not verify compat signature of hypercore v9"); - - remove_dir_all(dir).unwrap() - } -} - -#[test] -#[ignore] -fn compat_signatures_work() { - // Port from mafintosh/hypercore when the necessary features are implemented - unimplemented!(); -} - -#[test] -#[ignore] -fn deterministic_signatures_after_replication() { - // Port from mafintosh/hypercore when the necessary features are implemented - unimplemented!(); -} - -fn hex_bytes(hex: &str) -> Vec { - HEXLOWER.decode(hex.as_bytes()).unwrap() -} - -#[cfg(feature = "v9")] -fn storage_path>(dir: P, s: Store) -> PathBuf { - let filename = match s { - Store::Tree => "tree", - Store::Data => "data", - Store::Bitfield => "bitfield", - Store::Signatures => "signatures", - Store::Keypair => "key", - #[cfg(feature = "v10")] - Store::Oplog => "oplog", - }; - dir.as_ref().join(filename) -} - -#[cfg(feature = "v9")] -async fn mk_storage() -> (PathBuf, Storage) { - let temp_dir = tempfile::tempdir().unwrap(); - let dir = temp_dir.into_path(); - let storage = Storage::new( - |s| { - let dir = dir.clone(); - Box::pin(async move { RandomAccessDisk::open(storage_path(dir, s)).await }) - }, - false, - ) - .await - .unwrap(); - (dir, storage) -} - -#[cfg(feature = "v9")] -fn read_bytes>(dir: P, s: Store) -> Vec { - let mut f = File::open(storage_path(dir, s)).unwrap(); - let mut bytes = Vec::new(); - f.read_to_end(&mut bytes).unwrap(); - bytes -} - -#[cfg(feature = "v9")] -fn mk_keypair(keypair_bytes: &[u8], public_key: &[u8]) -> Keypair { - let keypair = Keypair::from_bytes(&keypair_bytes).unwrap(); - assert_eq!( - keypair.secret.as_bytes().as_ref(), - &keypair_bytes[..ed25519_dalek::SECRET_KEY_LENGTH] - ); - assert_eq!(keypair.public.as_bytes().as_ref(), public_key); - keypair -} diff --git a/tests/core.rs b/tests/core.rs index b46c19ca..41085471 100644 --- a/tests/core.rs +++ b/tests/core.rs @@ -1,11 +1,8 @@ -#![cfg(feature = "v10")] - -mod common; +pub mod common; use anyhow::Result; use common::get_test_key_pair; -use hypercore::{Hypercore, RequestBlock, Storage}; -use random_access_memory::RandomAccessMemory; +use hypercore::{Hypercore, Storage}; #[async_std::test] async fn hypercore_new() -> Result<()> { diff --git a/tests/feed.rs b/tests/feed.rs deleted file mode 100644 index 4c1e158c..00000000 --- a/tests/feed.rs +++ /dev/null @@ -1,358 +0,0 @@ -extern crate random_access_memory as ram; - -mod common; - -#[cfg(feature = "v9")] -use common::create_feed; -#[cfg(feature = "v9")] -use hypercore::{generate_keypair, Feed, NodeTrait, PublicKey, SecretKey, Storage}; -use random_access_storage::RandomAccess; -use std::env::temp_dir; -use std::fmt::Debug; -use std::fs; -use std::io::Write; - -#[async_std::test] -#[cfg(feature = "v9")] -async fn create_with_key() { - let keypair = generate_keypair(); - let storage = Storage::new_memory().await.unwrap(); - let _feed = Feed::builder(keypair.public, storage) - .secret_key(keypair.secret) - .build() - .await - .unwrap(); -} - -#[async_std::test] -#[cfg(feature = "v9")] -async fn display() { - let feed = create_feed(50).await.unwrap(); - let output = format!("{}", feed); - assert_eq!(output.len(), 61); -} - -#[async_std::test] -#[cfg(feature = "v9")] -/// Verify `.append()` and `.get()` work. -async fn set_get() { - let mut feed = create_feed(50).await.unwrap(); - feed.append(b"hello").await.unwrap(); - feed.append(b"world").await.unwrap(); - - assert_eq!(feed.get(0).await.unwrap(), Some(b"hello".to_vec())); - assert_eq!(feed.get(1).await.unwrap(), Some(b"world".to_vec())); -} - -#[async_std::test] -#[cfg(feature = "v9")] -async fn append() { - let mut feed = create_feed(50).await.unwrap(); - feed.append(br#"{"hello":"world"}"#).await.unwrap(); - feed.append(br#"{"hello":"mundo"}"#).await.unwrap(); - feed.append(br#"{"hello":"welt"}"#).await.unwrap(); - - assert_eq!(feed.len(), 3); - assert_eq!(feed.byte_len(), 50); - - assert_eq!( - feed.get(0).await.unwrap(), - Some(br#"{"hello":"world"}"#.to_vec()) - ); - assert_eq!( - feed.get(1).await.unwrap(), - Some(br#"{"hello":"mundo"}"#.to_vec()) - ); - assert_eq!( - feed.get(2).await.unwrap(), - Some(br#"{"hello":"welt"}"#.to_vec()) - ); -} - -#[async_std::test] -#[cfg(feature = "v9")] -/// Verify the `.root_hashes()` method returns the right nodes. -async fn root_hashes() { - // If no roots exist we should get an error. - let mut feed = create_feed(50).await.unwrap(); - let res = feed.root_hashes(0).await; - assert!(res.is_err()); - - // If 1 entry exists, [0] should be the root. - feed.append(b"data").await.unwrap(); - let roots = feed.root_hashes(0).await.unwrap(); - assert_eq!(roots.len(), 1); - assert_eq!(roots[0].index(), 0); - - // If we query out of bounds, we should get an error. - let res = feed.root_hashes(6).await; - assert!(res.is_err()); - - // If 3 entries exist, [2,4] should be the roots. - feed.append(b"data").await.unwrap(); - feed.append(b"data").await.unwrap(); - let roots = feed.root_hashes(2).await.unwrap(); - assert_eq!(roots.len(), 2); - assert_eq!(roots[0].index(), 1); - assert_eq!(roots[1].index(), 4); -} - -#[async_std::test] -#[cfg(feature = "v9")] -async fn verify() { - let mut feed = create_feed(50).await.unwrap(); - let (public, secret) = copy_keys(&feed); - let feed_bytes = secret.to_bytes().to_vec(); - let storage = Storage::new( - |_| Box::pin(async { Ok(ram::RandomAccessMemory::new(50)) }), - false, - ) - .await - .unwrap(); - let mut evil_feed = Feed::builder(public, storage) - .secret_key(secret) - .build() - .await - .unwrap(); - - let evil_bytes = match &feed.secret_key() { - Some(key) => key.to_bytes(), - None => panic!("no secret key found"), - }; - - // Verify the keys are the same. - assert_eq!(&feed_bytes, &evil_bytes.to_vec()); - - // Verify that the signature on a single feed is correct. - feed.append(b"test").await.unwrap(); - let sig = feed.signature(0).await.unwrap(); - feed.verify(0, &sig).await.unwrap(); - - // Verify that the signature between two different feeds is different. - evil_feed.append(b"t0st").await.unwrap(); - let res = evil_feed.verify(0, &sig).await; - assert!(res.is_err()); -} - -#[async_std::test] -#[cfg(feature = "v9")] -async fn put() { - let mut a = create_feed(50).await.unwrap(); - let (public, secret) = copy_keys(&a); - let storage = Storage::new( - |_| Box::pin(async { Ok(ram::RandomAccessMemory::new(50)) }), - false, - ) - .await - .unwrap(); - let mut b = Feed::builder(public, storage) - .secret_key(secret) - .build() - .await - .unwrap(); - - for _ in 0..10u8 { - a.append(b"foo").await.unwrap(); - } - - let proof = a.proof(0, true).await.unwrap(); - b.put(0, None, proof).await.expect("no error"); - let proof = a - .proof_with_digest(4, b.digest(4), true) - .await - .expect(".proof() index 4, digest 4"); - b.put(4, None, proof).await.unwrap(); -} - -#[async_std::test] -#[cfg(feature = "v9")] -/// Put data from one feed into another, while veryfing hashes. -/// I.e. manual replication between two feeds. -async fn put_with_data() { - // Create a writable feed. - let mut a = create_feed(50).await.unwrap(); - - // Create a second feed with the first feed's key. - let (public, secret) = copy_keys(&a); - let storage = Storage::new_memory().await.unwrap(); - let mut b = Feed::builder(public, storage) - .secret_key(secret) - .build() - .await - .unwrap(); - - // Append 4 blocks of data to the writable feed. - a.append(b"hi").await.unwrap(); - a.append(b"ola").await.unwrap(); - a.append(b"ahoj").await.unwrap(); - a.append(b"salut").await.unwrap(); - - for i in 0..4 { - // Generate a proof for the index. - // The `include_hash` argument has to be set to false. - let a_proof = a.proof(i, false).await.unwrap(); - // Get the data for the index. - let a_data = a.get(i).await.unwrap(); - - // Put the data into the other hypercore. - b.put(i, a_data.as_deref(), a_proof.clone()).await.unwrap(); - - // Load the data we've put. - let b_data = b.get(i).await.unwrap(); - - // Assert the data was put correctly. - assert!(a_data == b_data, "Data correct"); - } -} - -#[async_std::test] -#[cfg(feature = "v9")] -async fn create_with_storage() { - let storage = Storage::new_memory().await.unwrap(); - assert!( - Feed::with_storage(storage).await.is_ok(), - "Could not create a feed with a storage." - ); -} - -#[async_std::test] -#[cfg(feature = "v9")] -async fn create_with_stored_public_key() { - let mut storage = Storage::new_memory().await.unwrap(); - let keypair = generate_keypair(); - storage.write_public_key(&keypair.public).await.unwrap(); - assert!( - Feed::with_storage(storage).await.is_ok(), - "Could not create a feed with a stored public key." - ); -} - -#[async_std::test] -#[cfg(feature = "v9")] -async fn create_with_stored_keys() { - let mut storage = Storage::new_memory().await.unwrap(); - let keypair = generate_keypair(); - storage.write_public_key(&keypair.public).await.unwrap(); - storage.write_secret_key(&keypair.secret).await.unwrap(); - assert!( - Feed::with_storage(storage).await.is_ok(), - "Could not create a feed with a stored keypair." - ); -} - -#[cfg(feature = "v9")] -fn copy_keys( - feed: &Feed> + Debug + Send>, -) -> (PublicKey, SecretKey) { - match &feed.secret_key() { - Some(secret) => { - let secret = secret.to_bytes(); - let public = &feed.public_key().to_bytes(); - - let public = PublicKey::from_bytes(public).unwrap(); - let secret = SecretKey::from_bytes(&secret).unwrap(); - - (public, secret) - } - _ => panic!(": Could not access secret key"), - } -} - -#[async_std::test] -#[cfg(feature = "v9")] -async fn audit() { - let mut feed = create_feed(50).await.unwrap(); - feed.append(b"hello").await.unwrap(); - feed.append(b"world").await.unwrap(); - match feed.audit().await { - Ok(audit_report) => { - assert_eq!(audit_report.valid_blocks, 2); - assert_eq!(audit_report.invalid_blocks, 0); - } - Err(e) => { - panic!(e); - } - } -} - -#[async_std::test] -#[cfg(feature = "v9")] -async fn audit_bad_data() { - let mut dir = temp_dir(); - dir.push("audit_bad_data"); - let storage = Storage::new_disk(&dir, false).await.unwrap(); - let mut feed = Feed::with_storage(storage).await.unwrap(); - feed.append(b"hello").await.unwrap(); - feed.append(b"world").await.unwrap(); - let datapath = dir.join("data"); - let mut hypercore_data = fs::OpenOptions::new() - .write(true) - .open(datapath) - .expect("Unable to open the hypercore's data file!"); - hypercore_data - .write_all(b"yello") - .expect("Unable to corrupt the hypercore data file!"); - - match feed.audit().await { - Ok(audit_report) => { - assert_eq!(audit_report.valid_blocks, 1); - assert_eq!(audit_report.invalid_blocks, 1); - // Ensure that audit has cleared up the invalid block - match feed.audit().await { - Ok(audit_report) => { - assert_eq!( - audit_report.valid_blocks, 1, - "Audit did not clean up the invalid block!" - ); - assert_eq!( - audit_report.invalid_blocks, 0, - "Audit did not clean up the invalid block!" - ); - fs::remove_dir_all(dir) - .expect("Should be able to remove our temporary directory"); - } - Err(e) => { - fs::remove_dir_all(dir) - .expect("Should be able to remove our temporary directory"); - panic!(e); - } - } - } - Err(e) => { - fs::remove_dir_all(dir).expect("Should be able to remove our temporary directory"); - panic!(e); - } - } -} - -#[async_std::test] -#[cfg(feature = "v9")] -async fn try_open_missing_dir() { - use rand::distributions::Alphanumeric; - use rand::{thread_rng, Rng}; - - let rand_string: String = thread_rng().sample_iter(&Alphanumeric).take(5).collect(); - let mut dir = std::env::temp_dir(); - let path = format!("hypercore_rs_test/nonexistent_paths_test/{}", rand_string); - dir.push(path); - - if Feed::open(&dir).await.is_err() { - panic!("Opening nonexistent dir at a path should succeed"); - } - - if let Ok(d) = std::fs::metadata(dir) { - if !d.is_dir() { - panic!("Opening nonexistent dir at a path must create dir"); - } - } else { - panic!("Opening nonexistent dir at a path must create dir"); - } -} - -#[async_std::test] -#[cfg(feature = "v9")] -async fn try_open_file_as_dir() { - if Feed::open("Cargo.toml").await.is_ok() { - panic!("Opening path that points to a file must result in error"); - } -} diff --git a/tests/js_interop.rs b/tests/js_interop.rs index 9daf082a..da7589e0 100644 --- a/tests/js_interop.rs +++ b/tests/js_interop.rs @@ -1,10 +1,9 @@ -mod common; -mod js; +pub mod common; +pub mod js; use std::{path::Path, sync::Once}; use anyhow::Result; use common::{create_hypercore_hash, get_test_key_pair}; -#[cfg(feature = "v10")] use hypercore::{Hypercore, Storage}; use js::{cleanup, install, js_run_step, prepare_test_set}; use random_access_disk::RandomAccessDisk; @@ -28,7 +27,6 @@ fn init() { #[async_test] #[cfg_attr(not(feature = "js_interop_tests"), ignore)] -#[cfg(feature = "v10")] async fn js_interop_js_first() -> Result<()> { init(); let work_dir = prepare_test_set(TEST_SET_JS_FIRST); @@ -48,7 +46,6 @@ async fn js_interop_js_first() -> Result<()> { #[async_test] #[cfg_attr(not(feature = "js_interop_tests"), ignore)] -#[cfg(feature = "v10")] async fn js_interop_rs_first() -> Result<()> { init(); let work_dir = prepare_test_set(TEST_SET_RS_FIRST); @@ -66,13 +63,11 @@ async fn js_interop_rs_first() -> Result<()> { Ok(()) } -#[cfg(feature = "v10")] async fn step_1_create(work_dir: &str) -> Result<()> { create_hypercore(work_dir).await?; Ok(()) } -#[cfg(feature = "v10")] async fn step_2_append_hello_world(work_dir: &str) -> Result<()> { let mut hypercore = open_hypercore(work_dir).await?; let append_outcome = hypercore.append_batch(&[b"Hello", b"World"]).await?; @@ -81,7 +76,6 @@ async fn step_2_append_hello_world(work_dir: &str) -> Result<()> { Ok(()) } -#[cfg(feature = "v10")] async fn step_3_read_and_append_unflushed(work_dir: &str) -> Result<()> { let mut hypercore = open_hypercore(work_dir).await?; let hello = hypercore.get(0).await?; @@ -112,7 +106,6 @@ async fn step_3_read_and_append_unflushed(work_dir: &str) -> Result<()> { Ok(()) } -#[cfg(feature = "v10")] async fn step_4_append_with_flush(work_dir: &str) -> Result<()> { let mut hypercore = open_hypercore(work_dir).await?; for i in 0..5 { @@ -123,7 +116,6 @@ async fn step_4_append_with_flush(work_dir: &str) -> Result<()> { Ok(()) } -#[cfg(feature = "v10")] async fn step_5_clear_some(work_dir: &str) -> Result<()> { let mut hypercore = open_hypercore(work_dir).await?; hypercore.clear(5, 6).await?; @@ -143,7 +135,6 @@ async fn step_5_clear_some(work_dir: &str) -> Result<()> { Ok(()) } -#[cfg(feature = "v10")] async fn create_hypercore(work_dir: &str) -> Result> { let path = Path::new(work_dir).to_owned(); let key_pair = get_test_key_pair(); @@ -151,7 +142,6 @@ async fn create_hypercore(work_dir: &str) -> Result> Ok(Hypercore::new_with_key_pair(storage, key_pair).await?) } -#[cfg(feature = "v10")] async fn open_hypercore(work_dir: &str) -> Result> { let path = Path::new(work_dir).to_owned(); let storage = Storage::new_disk(&path, false).await?; diff --git a/tests/model.rs b/tests/model.rs index bf84acad..5e80f847 100644 --- a/tests/model.rs +++ b/tests/model.rs @@ -1,7 +1,5 @@ -mod common; +pub mod common; -#[cfg(feature = "v9")] -use common::create_feed; use quickcheck::{quickcheck, Arbitrary, Gen}; use rand::seq::SliceRandom; use rand::Rng; @@ -17,9 +15,6 @@ enum Op { Append { data: Vec, }, - #[cfg(feature = "v9")] - Verify, - #[cfg(feature = "v10")] Clear { len_divisor_for_start: u8, len_divisor_for_length: u8, @@ -42,9 +37,6 @@ impl Arbitrary for Op { } Op::Append { data } } - #[cfg(feature = "v9")] - 2 => Op::Verify, - #[cfg(feature = "v10")] 2 => { let len_divisor_for_start: u8 = g.gen_range(1, 17); let len_divisor_for_length: u8 = g.gen_range(1, 17); @@ -59,53 +51,12 @@ impl Arbitrary for Op { } quickcheck! { - - #[cfg(feature = "v9")] - fn implementation_matches_model(ops: Vec) -> bool { - async_std::task::block_on(async { - let page_size = 50; - - let mut insta = create_feed(page_size) - .await - .expect("Instance creation should be successful"); - let mut model = vec![]; - - for op in ops { - match op { - Op::Append { data } => { - insta.append(&data).await.expect("Append should be successful"); - model.push(data); - }, - Op::Get { index } => { - let data = insta.get(index).await.expect("Get should be successful"); - if index >= insta.len() { - assert_eq!(data, None); - } else { - assert_eq!(data, Some(model[index as usize].clone())); - } - }, - Op::Verify => { - let len = insta.len(); - if len == 0 { - insta.signature(len).await.unwrap_err(); - } else { - // Always test index of last entry, which is `len - 1`. - let len = len - 1; - let sig = insta.signature(len).await.expect("Signature should exist"); - insta.verify(len, &sig).await.expect("Signature should match"); - } - }, - } - } - true - }) - } - #[cfg(all(feature = "v10", feature = "async-std"))] + #[cfg(feature = "async-std")] fn implementation_matches_model(ops: Vec) -> bool { async_std::task::block_on(assert_implementation_matches_model(ops)) } - #[cfg(all(feature = "v10", feature = "tokio"))] + #[cfg(feature = "tokio")] fn implementation_matches_model(ops: Vec) -> bool { let rt = tokio::runtime::Runtime::new().unwrap(); rt.block_on(async { @@ -114,7 +65,6 @@ quickcheck! { } } -#[cfg(feature = "v10")] async fn assert_implementation_matches_model(ops: Vec) -> bool { use hypercore::{Hypercore, Storage}; diff --git a/tests/regression.rs b/tests/regression.rs deleted file mode 100644 index 6c5fc882..00000000 --- a/tests/regression.rs +++ /dev/null @@ -1,20 +0,0 @@ -mod common; - -#[cfg(feature = "v9")] -use common::create_feed; - -// Postmortem: errors were happening correctly, but the error check in -// `.signature()` was off. Instead of checking for a range (`<`), we were -// checking inclusively `<=`. All we had to do was fix the check, and we all -// good. -#[async_std::test] -#[cfg(feature = "v9")] -async fn regression_01() { - let mut feed = create_feed(50).await.unwrap(); - assert_eq!(feed.len(), 0); - feed.signature(0).await.unwrap_err(); - - let data = b"some_data"; - feed.append(data).await.unwrap(); - feed.signature(0).await.unwrap(); -} diff --git a/tests/storage.rs b/tests/storage.rs deleted file mode 100644 index 62bcf2e3..00000000 --- a/tests/storage.rs +++ /dev/null @@ -1,58 +0,0 @@ -#[cfg(feature = "v9")] -use ed25519_dalek::PublicKey; - -#[cfg(feature = "v9")] -use hypercore::{generate_keypair, sign, verify, Signature, Storage}; - -#[async_std::test] -#[cfg(feature = "v9")] -async fn should_write_and_read_keypair() { - let keypair = generate_keypair(); - let msg = b"hello"; - // prepare a signature - let sig: Signature = sign(&keypair.public, &keypair.secret, msg); - - let mut storage = Storage::new_memory().await.unwrap(); - assert!( - storage.write_secret_key(&keypair.secret).await.is_ok(), - "Can not store secret key." - ); - assert!( - storage.write_public_key(&keypair.public).await.is_ok(), - "Can not store public key." - ); - - let read = storage.read_public_key().await; - assert!(read.is_ok(), "Can not read public key"); - let public_key: PublicKey = read.unwrap(); - assert!(verify(&public_key, msg, Some(&sig)).is_ok()); -} - -#[async_std::test] -#[cfg(feature = "v9")] -async fn should_read_partial_keypair() { - let keypair = generate_keypair(); - let mut storage = Storage::new_memory().await.unwrap(); - assert!( - storage.write_public_key(&keypair.public).await.is_ok(), - "Can not store public key." - ); - - let partial = storage.read_partial_keypair().await.unwrap(); - assert!(partial.secret.is_none(), "A secret key is present"); -} - -#[async_std::test] -#[cfg(feature = "v9")] -async fn should_read_no_keypair() { - let mut storage = Storage::new_memory().await.unwrap(); - let partial = storage.read_partial_keypair().await; - assert!(partial.is_none(), "A key is present"); -} - -#[async_std::test] -#[cfg(feature = "v9")] -async fn should_read_empty_public_key() { - let mut storage = Storage::new_memory().await.unwrap(); - assert!(storage.read_public_key().await.is_err()); -} From fcd6dfed3587d6196abe9d0e782be3c23f7242ee Mon Sep 17 00:00:00 2001 From: Timo Tiuraniemi Date: Fri, 17 Mar 2023 09:27:38 +0200 Subject: [PATCH 096/157] Rename v10 modules --- src/{bitfield_v10 => bitfield}/dynamic.rs | 0 src/{bitfield_v10 => bitfield}/fixed.rs | 0 src/{bitfield_v10 => bitfield}/mod.rs | 0 src/core.rs | 4 ++-- src/lib.rs | 6 +++--- src/prelude.rs | 2 +- src/{storage_v10 => storage}/mod.rs | 0 7 files changed, 6 insertions(+), 6 deletions(-) rename src/{bitfield_v10 => bitfield}/dynamic.rs (100%) rename src/{bitfield_v10 => bitfield}/fixed.rs (100%) rename src/{bitfield_v10 => bitfield}/mod.rs (100%) rename src/{storage_v10 => storage}/mod.rs (100%) diff --git a/src/bitfield_v10/dynamic.rs b/src/bitfield/dynamic.rs similarity index 100% rename from src/bitfield_v10/dynamic.rs rename to src/bitfield/dynamic.rs diff --git a/src/bitfield_v10/fixed.rs b/src/bitfield/fixed.rs similarity index 100% rename from src/bitfield_v10/fixed.rs rename to src/bitfield/fixed.rs diff --git a/src/bitfield_v10/mod.rs b/src/bitfield/mod.rs similarity index 100% rename from src/bitfield_v10/mod.rs rename to src/bitfield/mod.rs diff --git a/src/core.rs b/src/core.rs index 9262c329..9ebff03a 100644 --- a/src/core.rs +++ b/src/core.rs @@ -1,12 +1,12 @@ //! Hypercore's main abstraction. Exposes an append-only, secure log structure. use crate::{ - bitfield_v10::Bitfield, + bitfield::Bitfield, common::{BitfieldUpdate, Proof, StoreInfo, ValuelessProof}, crypto::generate_keypair, data::BlockStore, oplog::{Header, Oplog, MAX_OPLOG_ENTRIES_BYTE_SIZE}, - storage_v10::{PartialKeypair, Storage}, + storage::{PartialKeypair, Storage}, tree::{MerkleTree, MerkleTreeChangeset}, RequestBlock, RequestSeek, RequestUpgrade, }; diff --git a/src/lib.rs b/src/lib.rs index 9b452cdb..a97d63fe 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -28,14 +28,14 @@ pub mod compact_encoding; pub mod prelude; -mod bitfield_v10; +mod bitfield; mod common; mod core; mod crypto; mod data; mod event; mod oplog; -mod storage_v10; +mod storage; mod tree; pub use crate::common::Node; @@ -46,7 +46,7 @@ pub use crate::common::{ pub use crate::core::Hypercore; pub use crate::crypto::{generate_keypair, sign, verify, Signature}; pub use crate::event::Event; -pub use crate::storage_v10::{PartialKeypair, Storage}; +pub use crate::storage::{PartialKeypair, Storage}; pub use ed25519_dalek::{ ExpandedSecretKey, Keypair, PublicKey, SecretKey, EXPANDED_SECRET_KEY_LENGTH, KEYPAIR_LENGTH, PUBLIC_KEY_LENGTH, SECRET_KEY_LENGTH, diff --git a/src/prelude.rs b/src/prelude.rs index f6bc2097..142f3648 100644 --- a/src/prelude.rs +++ b/src/prelude.rs @@ -1,4 +1,4 @@ //! Convenience wrapper to import all of Hypercore's core. pub use crate::common::Store; pub use crate::core::Hypercore; -pub use crate::storage_v10::{PartialKeypair, Storage}; +pub use crate::storage::{PartialKeypair, Storage}; diff --git a/src/storage_v10/mod.rs b/src/storage/mod.rs similarity index 100% rename from src/storage_v10/mod.rs rename to src/storage/mod.rs From ca252f4492186ee488c329c49dcc94d3191e4468 Mon Sep 17 00:00:00 2001 From: Timo Tiuraniemi Date: Fri, 17 Mar 2023 12:55:48 +0200 Subject: [PATCH 097/157] Use random-access-* with specialized error types --- Cargo.toml | 6 +++--- src/core.rs | 4 ++-- src/storage/mod.rs | 10 +++++++--- 3 files changed, 12 insertions(+), 8 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 3a840125..4a3940df 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -32,8 +32,8 @@ memory-pager = "0.9.0" merkle-tree-stream = "0.12.0" pretty-hash = "0.4.1" rand = "0.7.3" -random-access-memory = { git = "https://github.com/ttiurani/random-access-memory", branch = "tokio" } -random-access-storage = { git = "https://github.com/ttiurani/random-access-storage" , branch = "tokio"} +random-access-memory = { git = "https://github.com/ttiurani/random-access-memory", branch = "thiserror" } +random-access-storage = { git = "https://github.com/ttiurani/random-access-storage" , branch = "thiserror"} sha2 = "0.9.2" sleep-parser = "0.8.0" sparse-bitfield = "0.11.0" @@ -45,7 +45,7 @@ crc32fast = "1.3.2" intmap = "2.0.0" [target.'cfg(not(target_arch = "wasm32"))'.dependencies] -random-access-disk = { git = "https://github.com/ttiurani/random-access-disk", branch = "tokio" } +random-access-disk = { git = "https://github.com/ttiurani/random-access-disk", branch = "thiserror" } [dev-dependencies] quickcheck = "0.9.2" diff --git a/src/core.rs b/src/core.rs index 9ebff03a..097f8761 100644 --- a/src/core.rs +++ b/src/core.rs @@ -21,7 +21,7 @@ use std::fmt::Debug; #[derive(Debug)] pub struct Hypercore where - T: RandomAccess> + Debug, + T: RandomAccess + Debug, { pub(crate) key_pair: PartialKeypair, pub(crate) storage: Storage, @@ -52,7 +52,7 @@ pub struct Info { impl Hypercore where - T: RandomAccess> + Debug + Send, + T: RandomAccess + Debug + Send, { /// Creates new hypercore using given storage with random key pair pub async fn new(storage: Storage) -> Result> { diff --git a/src/storage/mod.rs b/src/storage/mod.rs index f16bca04..dadc1656 100644 --- a/src/storage/mod.rs +++ b/src/storage/mod.rs @@ -6,7 +6,7 @@ use futures::future::FutureExt; #[cfg(not(target_arch = "wasm32"))] use random_access_disk::RandomAccessDisk; use random_access_memory::RandomAccessMemory; -use random_access_storage::RandomAccess; +use random_access_storage::{RandomAccess, RandomAccessError}; use std::fmt::Debug; use std::path::PathBuf; @@ -51,12 +51,16 @@ where impl Storage where - T: RandomAccess> + Debug + Send, + T: RandomAccess + Debug + Send, { /// Create a new instance. Takes a callback to create new storage instances and overwrite flag. pub async fn open(create: Cb, overwrite: bool) -> Result where - Cb: Fn(Store) -> std::pin::Pin> + Send>>, + Cb: Fn( + Store, + ) -> std::pin::Pin< + Box> + Send>, + >, { let mut tree = create(Store::Tree).await?; let mut data = create(Store::Data).await?; From 074477fd3e05518cfa2326026256814e6c196a2c Mon Sep 17 00:00:00 2001 From: Timo Tiuraniemi Date: Wed, 22 Mar 2023 11:24:08 +0200 Subject: [PATCH 098/157] Move compact_encoding to its own crate Required implementing a wrapper struct for compact_encoding::State called HypercoreState, to be able to export the encodings to be used in hypercore-protocol --- Cargo.toml | 1 + src/compact_encoding/custom.rs | 198 ---------------- src/compact_encoding/generic.rs | 142 ------------ src/compact_encoding/mod.rs | 7 - src/compact_encoding/types.rs | 397 -------------------------------- src/crypto/hash.rs | 5 +- src/encoding.rs | 241 +++++++++++++++++++ src/lib.rs | 2 +- src/oplog/entry.rs | 71 +++--- src/oplog/header.rs | 3 +- src/oplog/mod.rs | 41 ++-- src/tree/merkle_tree.rs | 2 +- tests/compact_encoding.rs | 185 --------------- 13 files changed, 306 insertions(+), 989 deletions(-) delete mode 100644 src/compact_encoding/custom.rs delete mode 100644 src/compact_encoding/generic.rs delete mode 100644 src/compact_encoding/mod.rs delete mode 100644 src/compact_encoding/types.rs create mode 100644 src/encoding.rs delete mode 100644 tests/compact_encoding.rs diff --git a/Cargo.toml b/Cargo.toml index 4a3940df..d3d7191c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -26,6 +26,7 @@ blake2-rfc = "0.2.18" byteorder = "1.3.4" ed25519-dalek = "1.0.1" anyhow = "1.0.26" +compact-encoding = { git = "https://github.com/datrs/compact-encoding" } flat-tree = { git = "https://github.com/ttiurani/flat-tree", branch = "v10" } lazy_static = "1.4.0" memory-pager = "0.9.0" diff --git a/src/compact_encoding/custom.rs b/src/compact_encoding/custom.rs deleted file mode 100644 index b7ff0cc4..00000000 --- a/src/compact_encoding/custom.rs +++ /dev/null @@ -1,198 +0,0 @@ -//! Hypercore-specific compact encodings -use super::{CompactEncoding, State}; -use crate::{ - DataBlock, DataHash, DataSeek, DataUpgrade, Node, RequestBlock, RequestSeek, RequestUpgrade, -}; - -impl CompactEncoding for State { - fn preencode(&mut self, value: &Node) { - self.preencode(&value.index); - self.preencode(&value.length); - self.preencode_fixed_32(); - } - - fn encode(&mut self, value: &Node, buffer: &mut [u8]) { - self.encode(&value.index, buffer); - self.encode(&value.length, buffer); - self.encode_fixed_32(&value.hash, buffer); - } - - fn decode(&mut self, buffer: &[u8]) -> Node { - let index: u64 = self.decode(buffer); - let length: u64 = self.decode(buffer); - let hash: Box<[u8]> = self.decode_fixed_32(buffer); - Node::new(index, hash.to_vec(), length) - } -} - -impl CompactEncoding> for State { - fn preencode(&mut self, value: &Vec) { - let len = value.len(); - self.preencode(&len); - for val in value.into_iter() { - self.preencode(val); - } - } - - fn encode(&mut self, value: &Vec, buffer: &mut [u8]) { - let len = value.len(); - self.encode(&len, buffer); - for val in value { - self.encode(val, buffer); - } - } - - fn decode(&mut self, buffer: &[u8]) -> Vec { - let len: usize = self.decode(buffer); - let mut value = Vec::with_capacity(len); - for _ in 0..len { - value.push(self.decode(buffer)); - } - value - } -} - -impl CompactEncoding for State { - fn preencode(&mut self, value: &RequestBlock) { - self.preencode(&value.index); - self.preencode(&value.nodes); - } - - fn encode(&mut self, value: &RequestBlock, buffer: &mut [u8]) { - self.encode(&value.index, buffer); - self.encode(&value.nodes, buffer); - } - - fn decode(&mut self, buffer: &[u8]) -> RequestBlock { - let index: u64 = self.decode(buffer); - let nodes: u64 = self.decode(buffer); - RequestBlock { index, nodes } - } -} - -impl CompactEncoding for State { - fn preencode(&mut self, value: &RequestSeek) { - self.preencode(&value.bytes); - } - - fn encode(&mut self, value: &RequestSeek, buffer: &mut [u8]) { - self.encode(&value.bytes, buffer); - } - - fn decode(&mut self, buffer: &[u8]) -> RequestSeek { - let bytes: u64 = self.decode(buffer); - RequestSeek { bytes } - } -} - -impl CompactEncoding for State { - fn preencode(&mut self, value: &RequestUpgrade) { - self.preencode(&value.start); - self.preencode(&value.length); - } - - fn encode(&mut self, value: &RequestUpgrade, buffer: &mut [u8]) { - self.encode(&value.start, buffer); - self.encode(&value.length, buffer); - } - - fn decode(&mut self, buffer: &[u8]) -> RequestUpgrade { - let start: u64 = self.decode(buffer); - let length: u64 = self.decode(buffer); - RequestUpgrade { start, length } - } -} - -impl CompactEncoding for State { - fn preencode(&mut self, value: &DataBlock) { - self.preencode(&value.index); - self.preencode(&value.value); - self.preencode(&value.nodes); - } - - fn encode(&mut self, value: &DataBlock, buffer: &mut [u8]) { - self.encode(&value.index, buffer); - self.encode(&value.value, buffer); - self.encode(&value.nodes, buffer); - } - - fn decode(&mut self, buffer: &[u8]) -> DataBlock { - let index: u64 = self.decode(buffer); - let value: Vec = self.decode(buffer); - let nodes: Vec = self.decode(buffer); - DataBlock { - index, - value, - nodes, - } - } -} - -impl CompactEncoding for State { - fn preencode(&mut self, value: &DataHash) { - self.preencode(&value.index); - self.preencode(&value.nodes); - } - - fn encode(&mut self, value: &DataHash, buffer: &mut [u8]) { - self.encode(&value.index, buffer); - self.encode(&value.nodes, buffer); - } - - fn decode(&mut self, buffer: &[u8]) -> DataHash { - let index: u64 = self.decode(buffer); - let nodes: Vec = self.decode(buffer); - DataHash { index, nodes } - } -} - -impl CompactEncoding for State { - fn preencode(&mut self, value: &DataSeek) { - self.preencode(&value.bytes); - self.preencode(&value.nodes); - } - - fn encode(&mut self, value: &DataSeek, buffer: &mut [u8]) { - self.encode(&value.bytes, buffer); - self.encode(&value.nodes, buffer); - } - - fn decode(&mut self, buffer: &[u8]) -> DataSeek { - let bytes: u64 = self.decode(buffer); - let nodes: Vec = self.decode(buffer); - DataSeek { bytes, nodes } - } -} - -impl CompactEncoding for State { - fn preencode(&mut self, value: &DataUpgrade) { - self.preencode(&value.start); - self.preencode(&value.length); - self.preencode(&value.nodes); - self.preencode(&value.additional_nodes); - self.preencode(&value.signature); - } - - fn encode(&mut self, value: &DataUpgrade, buffer: &mut [u8]) { - self.encode(&value.start, buffer); - self.encode(&value.length, buffer); - self.encode(&value.nodes, buffer); - self.encode(&value.additional_nodes, buffer); - self.encode(&value.signature, buffer); - } - - fn decode(&mut self, buffer: &[u8]) -> DataUpgrade { - let start: u64 = self.decode(buffer); - let length: u64 = self.decode(buffer); - let nodes: Vec = self.decode(buffer); - let additional_nodes: Vec = self.decode(buffer); - let signature: Vec = self.decode(buffer); - DataUpgrade { - start, - length, - nodes, - additional_nodes, - signature, - } - } -} diff --git a/src/compact_encoding/generic.rs b/src/compact_encoding/generic.rs deleted file mode 100644 index 78fdea67..00000000 --- a/src/compact_encoding/generic.rs +++ /dev/null @@ -1,142 +0,0 @@ -//! Generic compact encodings -use super::{CompactEncoding, State}; - -impl CompactEncoding for State { - fn preencode(&mut self, value: &String) { - self.preencode_str(value) - } - - fn encode(&mut self, value: &String, buffer: &mut [u8]) { - self.encode_str(value, buffer) - } - - fn decode(&mut self, buffer: &[u8]) -> String { - self.decode_string(buffer) - } -} - -impl CompactEncoding for State { - fn preencode(&mut self, _: &u8) { - self.end += 1; - } - - fn encode(&mut self, value: &u8, buffer: &mut [u8]) { - buffer[self.start] = *value; - self.start += 1; - } - - fn decode(&mut self, buffer: &[u8]) -> u8 { - let value = buffer[self.start]; - self.start += 1; - value - } -} - -impl CompactEncoding for State { - fn preencode(&mut self, value: &u32) { - self.preencode_uint_var(value) - } - - fn encode(&mut self, value: &u32, buffer: &mut [u8]) { - self.encode_u32_var(value, buffer) - } - - fn decode(&mut self, buffer: &[u8]) -> u32 { - self.decode_u32_var(buffer) - } -} - -impl CompactEncoding for State { - fn preencode(&mut self, value: &u64) { - self.preencode_uint_var(value) - } - - fn encode(&mut self, value: &u64, buffer: &mut [u8]) { - self.encode_u64_var(value, buffer) - } - - fn decode(&mut self, buffer: &[u8]) -> u64 { - self.decode_u64_var(buffer) - } -} - -impl CompactEncoding for State { - fn preencode(&mut self, value: &usize) { - self.preencode_usize_var(value) - } - - fn encode(&mut self, value: &usize, buffer: &mut [u8]) { - self.encode_usize_var(value, buffer) - } - - fn decode(&mut self, buffer: &[u8]) -> usize { - self.decode_usize_var(buffer) - } -} - -impl CompactEncoding> for State { - fn preencode(&mut self, value: &Box<[u8]>) { - self.preencode_buffer(value); - } - - fn encode(&mut self, value: &Box<[u8]>, buffer: &mut [u8]) { - self.encode_buffer(value, buffer); - } - - fn decode(&mut self, buffer: &[u8]) -> Box<[u8]> { - self.decode_buffer(buffer) - } -} - -impl CompactEncoding> for State { - fn preencode(&mut self, value: &Vec) { - self.preencode_buffer_vec(value); - } - - fn encode(&mut self, value: &Vec, buffer: &mut [u8]) { - self.encode_buffer(value, buffer); - } - - fn decode(&mut self, buffer: &[u8]) -> Vec { - self.decode_buffer_vec(buffer).to_vec() - } -} - -impl CompactEncoding> for State { - fn preencode(&mut self, value: &Vec) { - let len = value.len(); - self.preencode_usize_var(&len); - self.end += len * 4; - } - - fn encode(&mut self, value: &Vec, buffer: &mut [u8]) { - let len = value.len(); - self.encode_usize_var(&len, buffer); - for entry in value { - self.encode_u32(*entry, buffer); - } - } - - fn decode(&mut self, buffer: &[u8]) -> Vec { - let len = self.decode_usize_var(buffer); - let mut value: Vec = Vec::with_capacity(len); - for _ in 0..len { - value.push(self.decode_u32(&buffer)); - } - value - } -} - -impl CompactEncoding> for State { - fn preencode(&mut self, value: &Vec) { - self.preencode_string_array(value); - } - - fn encode(&mut self, value: &Vec, buffer: &mut [u8]) { - self.encode_string_array(value, buffer); - } - - fn decode(&mut self, buffer: &[u8]) -> Vec { - self.decode_string_array(buffer) - } -} diff --git a/src/compact_encoding/mod.rs b/src/compact_encoding/mod.rs deleted file mode 100644 index 3a75d5c5..00000000 --- a/src/compact_encoding/mod.rs +++ /dev/null @@ -1,7 +0,0 @@ -//! Compact encoding module. Rust implementation of https://github.com/compact-encoding/compact-encoding. - -pub mod custom; -pub mod generic; -pub mod types; - -pub use types::{CompactEncoding, State}; diff --git a/src/compact_encoding/types.rs b/src/compact_encoding/types.rs deleted file mode 100644 index 90e51a77..00000000 --- a/src/compact_encoding/types.rs +++ /dev/null @@ -1,397 +0,0 @@ -//! Basic types of compact_encoding. -use std::convert::TryFrom; -use std::fmt::Debug; - -const U16_SIGNIFIER: u8 = 0xfd; -const U32_SIGNIFIER: u8 = 0xfe; -const U64_SIGNIFIER: u8 = 0xff; - -/// State. -#[derive(Debug, Clone)] -pub struct State { - /// Start position - pub start: usize, - /// End position - pub end: usize, -} - -impl State { - /// Create emtpy state - pub fn new() -> State { - State::new_with_start_and_end(0, 0) - } - - /// Create a state with an already known size. - /// With this, you can/must skip the preencode step. - pub fn new_with_size(size: usize) -> (State, Box<[u8]>) { - ( - State::new_with_start_and_end(0, size), - vec![0; size].into_boxed_slice(), - ) - } - - /// Create a state with a start and end already known. - pub fn new_with_start_and_end(start: usize, end: usize) -> State { - State { start, end } - } - - /// Create a state from existing buffer. - pub fn from_buffer(buffer: &[u8]) -> State { - State::new_with_start_and_end(0, buffer.len()) - } - - /// After calling preencode(), this allocates the right size buffer to the heap. - /// Follow this with the same number of encode() steps to fill the created buffer. - pub fn create_buffer(&self) -> Box<[u8]> { - vec![0; self.end].into_boxed_slice() - } - - /// Preencode a string slice - pub fn preencode_str(&mut self, value: &str) { - self.preencode_usize_var(&value.len()); - self.end += value.len(); - } - - /// Encode a string slice - pub fn encode_str(&mut self, value: &str, buffer: &mut [u8]) { - let len = value.len(); - self.encode_usize_var(&len, buffer); - buffer[self.start..self.start + len].copy_from_slice(value.as_bytes()); - self.start += len; - } - - /// Decode a String - pub fn decode_string(&mut self, buffer: &[u8]) -> String { - let len = self.decode_usize_var(buffer); - let value = std::str::from_utf8(&buffer[self.start..self.start + len]) - .expect("string is invalid UTF-8"); - self.start += len; - value.to_string() - } - - /// Preencode a variable length usigned int - pub fn preencode_uint_var + Ord>(&mut self, uint: &T) { - self.end += if *uint < T::from(U16_SIGNIFIER.into()) { - 1 - } else if *uint <= T::from(0xffff) { - 3 - } else if *uint <= T::from(0xffffffff) { - 5 - } else { - 9 - }; - } - - /// Decode a fixed length u8 - pub fn decode_u8(&mut self, buffer: &[u8]) -> u8 { - let value: u8 = buffer[self.start]; - self.start += 1; - value - } - - /// Decode a fixed length u16 - pub fn decode_u16(&mut self, buffer: &[u8]) -> u16 { - let value: u16 = - ((buffer[self.start] as u16) << 0) | ((buffer[self.start + 1] as u16) << 8); - self.start += 2; - value - } - - /// Encode a variable length u32 - pub fn encode_u32_var(&mut self, value: &u32, buffer: &mut [u8]) { - if *value < U16_SIGNIFIER.into() { - let bytes = value.to_le_bytes(); - buffer[self.start] = bytes[0]; - self.start += 1; - } else if *value <= 0xffff { - buffer[self.start] = U16_SIGNIFIER; - self.start += 1; - self.encode_uint16_bytes(&value.to_le_bytes(), buffer); - } else { - buffer[self.start] = U32_SIGNIFIER; - self.start += 1; - self.encode_u32(*value, buffer); - } - } - - /// Encode u32 to 4 LE bytes. - pub fn encode_u32(&mut self, uint: u32, buffer: &mut [u8]) { - self.encode_uint32_bytes(&uint.to_le_bytes(), buffer); - } - - /// Decode a variable length u32 - pub fn decode_u32_var(&mut self, buffer: &[u8]) -> u32 { - let first = buffer[self.start]; - self.start += 1; - if first < U16_SIGNIFIER { - first.into() - } else if first == U16_SIGNIFIER { - self.decode_u16(buffer).into() - } else { - self.decode_u32(buffer).into() - } - } - - /// Decode a fixed length u32 - pub fn decode_u32(&mut self, buffer: &[u8]) -> u32 { - let value: u32 = ((buffer[self.start] as u32) << 0) - | ((buffer[self.start + 1] as u32) << 8) - | ((buffer[self.start + 2] as u32) << 16) - | ((buffer[self.start + 3] as u32) << 24); - self.start += 4; - value - } - - /// Encode a variable length u64 - pub fn encode_u64_var(&mut self, value: &u64, buffer: &mut [u8]) { - if *value < U16_SIGNIFIER.into() { - let bytes = value.to_le_bytes(); - buffer[self.start] = bytes[0]; - self.start += 1; - } else if *value <= 0xffff { - buffer[self.start] = U16_SIGNIFIER; - self.start += 1; - self.encode_uint16_bytes(&value.to_le_bytes(), buffer); - } else if *value <= 0xffffffff { - buffer[self.start] = U32_SIGNIFIER; - self.start += 1; - self.encode_uint32_bytes(&value.to_le_bytes(), buffer); - } else { - buffer[self.start] = U64_SIGNIFIER; - self.start += 1; - self.encode_uint64_bytes(&value.to_le_bytes(), buffer); - } - } - - /// Encode u64 to 8 LE bytes. - pub fn encode_u64(&mut self, uint: u64, buffer: &mut [u8]) { - self.encode_uint64_bytes(&uint.to_le_bytes(), buffer); - } - - /// Decode a variable length u64 - pub fn decode_u64_var(&mut self, buffer: &[u8]) -> u64 { - let first = buffer[self.start]; - self.start += 1; - if first < U16_SIGNIFIER { - first.into() - } else if first == U16_SIGNIFIER { - self.decode_u16(buffer).into() - } else if first == U32_SIGNIFIER { - self.decode_u32(buffer).into() - } else { - self.decode_u64(buffer) - } - } - - /// Decode a fixed length u64 - pub fn decode_u64(&mut self, buffer: &[u8]) -> u64 { - let value: u64 = ((buffer[self.start] as u64) << 0) - | ((buffer[self.start + 1] as u64) << 8) - | ((buffer[self.start + 2] as u64) << 16) - | ((buffer[self.start + 3] as u64) << 24) - | ((buffer[self.start + 4] as u64) << 32) - | ((buffer[self.start + 5] as u64) << 40) - | ((buffer[self.start + 6] as u64) << 48) - | ((buffer[self.start + 7] as u64) << 56); - self.start += 8; - value - } - - /// Preencode a byte buffer - pub fn preencode_buffer(&mut self, value: &Box<[u8]>) { - let len = value.len(); - self.preencode_usize_var(&len); - self.end += len; - } - - /// Preencode a vector byte buffer - pub fn preencode_buffer_vec(&mut self, value: &Vec) { - let len = value.len(); - self.preencode_usize_var(&len); - self.end += len; - } - - /// Encode a byte buffer - pub fn encode_buffer(&mut self, value: &[u8], buffer: &mut [u8]) { - let len = value.len(); - self.encode_usize_var(&len, buffer); - buffer[self.start..self.start + len].copy_from_slice(value); - self.start += len; - } - - /// Decode a byte buffer - pub fn decode_buffer(&mut self, buffer: &[u8]) -> Box<[u8]> { - self.decode_buffer_vec(buffer).into_boxed_slice() - } - - /// Decode a vector byte buffer - pub fn decode_buffer_vec(&mut self, buffer: &[u8]) -> Vec { - let len = self.decode_usize_var(buffer); - let value = buffer[self.start..self.start + len].to_vec(); - self.start += value.len(); - value - } - - /// Preencode a raw byte buffer. Only possible to use if this is the last value - /// of the State. - pub fn preencode_raw_buffer(&mut self, value: &Vec) { - self.end += value.len(); - } - - /// Encode a raw byte buffer. Only possible to use if this is the last value - /// of the State. - pub fn encode_raw_buffer(&mut self, value: &[u8], buffer: &mut [u8]) { - buffer[self.start..self.start + value.len()].copy_from_slice(value); - self.start += value.len(); - } - - /// Decode a raw byte buffer. Only possible to use if this is the last value - /// of the State. - pub fn decode_raw_buffer(&mut self, buffer: &[u8]) -> Vec { - let value = buffer[self.start..self.end].to_vec(); - self.start = self.end; - value - } - - /// Preencode a fixed 32 byte buffer - pub fn preencode_fixed_32(&mut self) { - self.end += 32; - } - - /// Encode a fixed 32 byte buffer - pub fn encode_fixed_32(&mut self, value: &[u8], buffer: &mut [u8]) { - buffer[self.start..self.start + 32].copy_from_slice(value); - self.start += 32; - } - - /// Decode a fixed 32 byte buffer - pub fn decode_fixed_32(&mut self, buffer: &[u8]) -> Box<[u8]> { - let value = buffer[self.start..self.start + 32] - .to_vec() - .into_boxed_slice(); - self.start += 32; - value - } - - /// Preencode a string array - pub fn preencode_string_array(&mut self, value: &Vec) { - let len = value.len(); - self.preencode_usize_var(&len); - for string_value in value.into_iter() { - self.preencode_str(string_value); - } - } - - /// Encode a String array - pub fn encode_string_array(&mut self, value: &Vec, buffer: &mut [u8]) { - let len = value.len(); - self.encode_usize_var(&len, buffer); - for string_value in value { - self.encode_str(string_value, buffer); - } - } - - /// Decode a String array - pub fn decode_string_array(&mut self, buffer: &[u8]) -> Vec { - let len = self.decode_usize_var(buffer); - let mut value = Vec::with_capacity(len); - for _ in 0..len { - value.push(self.decode_string(buffer)); - } - value - } - - /// Encode a 2 byte unsigned integer - pub fn encode_uint16_bytes(&mut self, bytes: &[u8], buffer: &mut [u8]) { - buffer[self.start] = bytes[0]; - buffer[self.start + 1] = bytes[1]; - self.start += 2; - } - - /// Encode a 4 byte unsigned integer - pub fn encode_uint32_bytes(&mut self, bytes: &[u8], buffer: &mut [u8]) { - self.encode_uint16_bytes(bytes, buffer); - buffer[self.start] = bytes[2]; - buffer[self.start + 1] = bytes[3]; - self.start += 2; - } - - /// Encode an 8 byte unsigned integer - pub fn encode_uint64_bytes(&mut self, bytes: &[u8], buffer: &mut [u8]) { - self.encode_uint32_bytes(bytes, buffer); - buffer[self.start] = bytes[4]; - buffer[self.start + 1] = bytes[5]; - buffer[self.start + 2] = bytes[6]; - buffer[self.start + 3] = bytes[7]; - self.start += 4; - } - - /// Preencode a variable length usize - pub fn preencode_usize_var(&mut self, value: &usize) { - // TODO: This repeats the logic from above that works for u8 -> u64, but sadly not usize - self.end += if *value < U16_SIGNIFIER.into() { - 1 - } else if *value <= 0xffff { - 3 - } else if *value <= 0xffffffff { - 5 - } else { - 9 - }; - } - - /// Encode a variable length usize - pub fn encode_usize_var(&mut self, value: &usize, buffer: &mut [u8]) { - if *value <= 0xfc { - let bytes = value.to_le_bytes(); - buffer[self.start] = bytes[0]; - self.start += 1; - } else if *value <= 0xffff { - buffer[self.start] = U16_SIGNIFIER; - self.start += 1; - self.encode_uint16_bytes(&value.to_le_bytes(), buffer); - } else if *value <= 0xffffffff { - buffer[self.start] = U32_SIGNIFIER; - self.start += 1; - self.encode_uint32_bytes(&value.to_le_bytes(), buffer); - } else { - buffer[self.start] = U64_SIGNIFIER; - self.start += 1; - self.encode_uint64_bytes(&value.to_le_bytes(), buffer); - } - } - - /// Decode a variable length usize - pub fn decode_usize_var(&mut self, buffer: &[u8]) -> usize { - let first = buffer[self.start]; - self.start += 1; - // NB: the from_le_bytes needs a [u8; 2] and that can't be efficiently - // created from a byte slice. - if first < U16_SIGNIFIER { - first.into() - } else if first == U16_SIGNIFIER { - self.decode_u16(buffer).into() - } else if first == U32_SIGNIFIER { - usize::try_from(self.decode_u32(buffer)) - .expect("Attempted converting to a 32 bit usize on below 32 bit system") - } else { - usize::try_from(self.decode_u64(buffer)) - .expect("Attempted converting to a 64 bit usize on below 64 bit system") - } - } -} - -/// Compact Encoding -pub trait CompactEncoding -where - T: Debug, -{ - /// Preencode - fn preencode(&mut self, value: &T); - - /// Encode - fn encode(&mut self, value: &T, buffer: &mut [u8]); - - /// Decode - fn decode(&mut self, buffer: &[u8]) -> T; -} diff --git a/src/crypto/hash.rs b/src/crypto/hash.rs index c33417e9..4579ffb1 100644 --- a/src/crypto/hash.rs +++ b/src/crypto/hash.rs @@ -1,15 +1,16 @@ pub use blake2_rfc::blake2b::Blake2bResult; -use crate::common::Node; -use crate::compact_encoding::State; use blake2_rfc::blake2b::Blake2b; use byteorder::{BigEndian, WriteBytesExt}; +use compact_encoding::State; use ed25519_dalek::PublicKey; use merkle_tree_stream::Node as NodeTrait; use std::convert::AsRef; use std::mem; use std::ops::{Deref, DerefMut}; +use crate::common::Node; + // https://en.wikipedia.org/wiki/Merkle_tree#Second_preimage_attack const LEAF_TYPE: [u8; 1] = [0x00]; const PARENT_TYPE: [u8; 1] = [0x01]; diff --git a/src/encoding.rs b/src/encoding.rs new file mode 100644 index 00000000..1e25bd9f --- /dev/null +++ b/src/encoding.rs @@ -0,0 +1,241 @@ +//! Hypercore-specific compact encodings +pub use compact_encoding::{CompactEncoding, State}; +use std::ops::{Deref, DerefMut}; + +use crate::{ + DataBlock, DataHash, DataSeek, DataUpgrade, Node, RequestBlock, RequestSeek, RequestUpgrade, +}; + +#[derive(Debug, Clone)] +/// Wrapper struct for compact_encoding::State +pub struct HypercoreState(pub State); + +impl HypercoreState { + /// Passthrought to compact_encoding + pub fn new() -> HypercoreState { + HypercoreState(State::new()) + } + + /// Passthrought to compact_encoding + pub fn new_with_size(size: usize) -> (HypercoreState, Box<[u8]>) { + let (state, buffer) = State::new_with_size(size); + (HypercoreState(state), buffer) + } + + /// Passthrought to compact_encoding + pub fn new_with_start_and_end(start: usize, end: usize) -> HypercoreState { + HypercoreState(State::new_with_start_and_end(start, end)) + } + + /// Passthrought to compact_encoding + pub fn from_buffer(buffer: &[u8]) -> HypercoreState { + HypercoreState(State::from_buffer(buffer)) + } +} + +impl Deref for HypercoreState { + type Target = State; + + fn deref(&self) -> &Self::Target { + &self.0 + } +} + +impl DerefMut for HypercoreState { + fn deref_mut(&mut self) -> &mut State { + &mut self.0 + } +} + +impl CompactEncoding for HypercoreState { + fn preencode(&mut self, value: &Node) { + self.0.preencode(&value.index); + self.0.preencode(&value.length); + self.0.preencode_fixed_32(); + } + + fn encode(&mut self, value: &Node, buffer: &mut [u8]) { + self.0.encode(&value.index, buffer); + self.0.encode(&value.length, buffer); + self.0.encode_fixed_32(&value.hash, buffer); + } + + fn decode(&mut self, buffer: &[u8]) -> Node { + let index: u64 = self.0.decode(buffer); + let length: u64 = self.0.decode(buffer); + let hash: Box<[u8]> = self.0.decode_fixed_32(buffer); + Node::new(index, hash.to_vec(), length) + } +} + +impl CompactEncoding> for HypercoreState { + fn preencode(&mut self, value: &Vec) { + let len = value.len(); + self.0.preencode(&len); + for val in value.into_iter() { + self.preencode(val); + } + } + + fn encode(&mut self, value: &Vec, buffer: &mut [u8]) { + let len = value.len(); + self.0.encode(&len, buffer); + for val in value { + self.encode(val, buffer); + } + } + + fn decode(&mut self, buffer: &[u8]) -> Vec { + let len: usize = self.0.decode(buffer); + let mut value = Vec::with_capacity(len); + for _ in 0..len { + value.push(self.decode(buffer)); + } + value + } +} + +impl CompactEncoding for HypercoreState { + fn preencode(&mut self, value: &RequestBlock) { + self.0.preencode(&value.index); + self.0.preencode(&value.nodes); + } + + fn encode(&mut self, value: &RequestBlock, buffer: &mut [u8]) { + self.0.encode(&value.index, buffer); + self.0.encode(&value.nodes, buffer); + } + + fn decode(&mut self, buffer: &[u8]) -> RequestBlock { + let index: u64 = self.0.decode(buffer); + let nodes: u64 = self.0.decode(buffer); + RequestBlock { index, nodes } + } +} + +impl CompactEncoding for HypercoreState { + fn preencode(&mut self, value: &RequestSeek) { + self.0.preencode(&value.bytes); + } + + fn encode(&mut self, value: &RequestSeek, buffer: &mut [u8]) { + self.0.encode(&value.bytes, buffer); + } + + fn decode(&mut self, buffer: &[u8]) -> RequestSeek { + let bytes: u64 = self.0.decode(buffer); + RequestSeek { bytes } + } +} + +impl CompactEncoding for HypercoreState { + fn preencode(&mut self, value: &RequestUpgrade) { + self.0.preencode(&value.start); + self.0.preencode(&value.length); + } + + fn encode(&mut self, value: &RequestUpgrade, buffer: &mut [u8]) { + self.0.encode(&value.start, buffer); + self.0.encode(&value.length, buffer); + } + + fn decode(&mut self, buffer: &[u8]) -> RequestUpgrade { + let start: u64 = self.0.decode(buffer); + let length: u64 = self.0.decode(buffer); + RequestUpgrade { start, length } + } +} + +impl CompactEncoding for HypercoreState { + fn preencode(&mut self, value: &DataBlock) { + self.0.preencode(&value.index); + self.0.preencode(&value.value); + self.preencode(&value.nodes); + } + + fn encode(&mut self, value: &DataBlock, buffer: &mut [u8]) { + self.0.encode(&value.index, buffer); + self.0.encode(&value.value, buffer); + self.encode(&value.nodes, buffer); + } + + fn decode(&mut self, buffer: &[u8]) -> DataBlock { + let index: u64 = self.0.decode(buffer); + let value: Vec = self.0.decode(buffer); + let nodes: Vec = self.decode(buffer); + DataBlock { + index, + value, + nodes, + } + } +} + +impl CompactEncoding for HypercoreState { + fn preencode(&mut self, value: &DataHash) { + self.0.preencode(&value.index); + self.preencode(&value.nodes); + } + + fn encode(&mut self, value: &DataHash, buffer: &mut [u8]) { + self.0.encode(&value.index, buffer); + self.encode(&value.nodes, buffer); + } + + fn decode(&mut self, buffer: &[u8]) -> DataHash { + let index: u64 = self.0.decode(buffer); + let nodes: Vec = self.decode(buffer); + DataHash { index, nodes } + } +} + +impl CompactEncoding for HypercoreState { + fn preencode(&mut self, value: &DataSeek) { + self.0.preencode(&value.bytes); + self.preencode(&value.nodes); + } + + fn encode(&mut self, value: &DataSeek, buffer: &mut [u8]) { + self.0.encode(&value.bytes, buffer); + self.encode(&value.nodes, buffer); + } + + fn decode(&mut self, buffer: &[u8]) -> DataSeek { + let bytes: u64 = self.0.decode(buffer); + let nodes: Vec = self.decode(buffer); + DataSeek { bytes, nodes } + } +} + +impl CompactEncoding for HypercoreState { + fn preencode(&mut self, value: &DataUpgrade) { + self.0.preencode(&value.start); + self.0.preencode(&value.length); + self.preencode(&value.nodes); + self.preencode(&value.additional_nodes); + self.0.preencode(&value.signature); + } + + fn encode(&mut self, value: &DataUpgrade, buffer: &mut [u8]) { + self.0.encode(&value.start, buffer); + self.0.encode(&value.length, buffer); + self.encode(&value.nodes, buffer); + self.encode(&value.additional_nodes, buffer); + self.0.encode(&value.signature, buffer); + } + + fn decode(&mut self, buffer: &[u8]) -> DataUpgrade { + let start: u64 = self.0.decode(buffer); + let length: u64 = self.0.decode(buffer); + let nodes: Vec = self.decode(buffer); + let additional_nodes: Vec = self.decode(buffer); + let signature: Vec = self.0.decode(buffer); + DataUpgrade { + start, + length, + nodes, + additional_nodes, + signature, + } + } +} diff --git a/src/lib.rs b/src/lib.rs index a97d63fe..96e0ab87 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -25,7 +25,7 @@ //! [Dat]: https://github.com/datrs //! [Feed]: crate::feed::Feed -pub mod compact_encoding; +pub mod encoding; pub mod prelude; mod bitfield; diff --git a/src/oplog/entry.rs b/src/oplog/entry.rs index eeb0ce1a..a74a7254 100644 --- a/src/oplog/entry.rs +++ b/src/oplog/entry.rs @@ -1,8 +1,5 @@ -use crate::{ - common::BitfieldUpdate, - compact_encoding::{CompactEncoding, State}, - Node, -}; +use crate::encoding::{CompactEncoding, HypercoreState}; +use crate::{common::BitfieldUpdate, Node}; /// Entry tree upgrade #[derive(Debug)] @@ -13,26 +10,26 @@ pub struct EntryTreeUpgrade { pub(crate) signature: Box<[u8]>, } -impl CompactEncoding for State { +impl CompactEncoding for HypercoreState { fn preencode(&mut self, value: &EntryTreeUpgrade) { - self.preencode(&value.fork); - self.preencode(&value.ancestors); - self.preencode(&value.length); - self.preencode(&value.signature); + self.0.preencode(&value.fork); + self.0.preencode(&value.ancestors); + self.0.preencode(&value.length); + self.0.preencode(&value.signature); } fn encode(&mut self, value: &EntryTreeUpgrade, buffer: &mut [u8]) { - self.encode(&value.fork, buffer); - self.encode(&value.ancestors, buffer); - self.encode(&value.length, buffer); - self.encode(&value.signature, buffer); + self.0.encode(&value.fork, buffer); + self.0.encode(&value.ancestors, buffer); + self.0.encode(&value.length, buffer); + self.0.encode(&value.signature, buffer); } fn decode(&mut self, buffer: &[u8]) -> EntryTreeUpgrade { - let fork: u64 = self.decode(buffer); - let ancestors: u64 = self.decode(buffer); - let length: u64 = self.decode(buffer); - let signature: Box<[u8]> = self.decode(buffer); + let fork: u64 = self.0.decode(buffer); + let ancestors: u64 = self.0.decode(buffer); + let length: u64 = self.0.decode(buffer); + let signature: Box<[u8]> = self.0.decode(buffer); EntryTreeUpgrade { fork, ancestors, @@ -42,25 +39,25 @@ impl CompactEncoding for State { } } -impl CompactEncoding for State { +impl CompactEncoding for HypercoreState { fn preencode(&mut self, value: &BitfieldUpdate) { - self.end += 1; - self.preencode(&value.start); - self.preencode(&value.length); + self.0.end += 1; + self.0.preencode(&value.start); + self.0.preencode(&value.length); } fn encode(&mut self, value: &BitfieldUpdate, buffer: &mut [u8]) { let flags: u8 = if value.drop { 1 } else { 0 }; - buffer[self.start] = flags; - self.start += 1; - self.encode(&value.start, buffer); - self.encode(&value.length, buffer); + buffer[self.0.start] = flags; + self.0.start += 1; + self.0.encode(&value.start, buffer); + self.0.encode(&value.length, buffer); } fn decode(&mut self, buffer: &[u8]) -> BitfieldUpdate { - let flags = self.decode_u8(buffer); - let start: u64 = self.decode(buffer); - let length: u64 = self.decode(buffer); + let flags = self.0.decode_u8(buffer); + let start: u64 = self.0.decode(buffer); + let length: u64 = self.0.decode(buffer); BitfieldUpdate { drop: flags == 1, start, @@ -79,11 +76,11 @@ pub struct Entry { pub(crate) bitfield: Option, } -impl CompactEncoding for State { +impl CompactEncoding for HypercoreState { fn preencode(&mut self, value: &Entry) { - self.end += 1; // flags + self.0.end += 1; // flags if value.user_data.len() > 0 { - self.preencode(&value.user_data); + self.0.preencode(&value.user_data); } if value.tree_nodes.len() > 0 { self.preencode(&value.tree_nodes); @@ -97,12 +94,12 @@ impl CompactEncoding for State { } fn encode(&mut self, value: &Entry, buffer: &mut [u8]) { - let start = self.start; - self.start += 1; + let start = self.0.start; + self.0.start += 1; let mut flags: u8 = 0; if value.user_data.len() > 0 { flags = flags | 1; - self.encode(&value.user_data, buffer); + self.0.encode(&value.user_data, buffer); } if value.tree_nodes.len() > 0 { flags = flags | 2; @@ -121,9 +118,9 @@ impl CompactEncoding for State { } fn decode(&mut self, buffer: &[u8]) -> Entry { - let flags = self.decode_u8(buffer); + let flags = self.0.decode_u8(buffer); let user_data: Vec = if flags & 1 != 0 { - self.decode(buffer) + self.0.decode(buffer) } else { vec![] }; diff --git a/src/oplog/header.rs b/src/oplog/header.rs index b37deb3d..c2f02754 100644 --- a/src/oplog/header.rs +++ b/src/oplog/header.rs @@ -1,4 +1,5 @@ -use crate::compact_encoding::{CompactEncoding, State}; +use compact_encoding::{CompactEncoding, State}; + use crate::crypto::{PublicKey, SecretKey}; use crate::PartialKeypair; diff --git a/src/oplog/mod.rs b/src/oplog/mod.rs index 64f8d47d..3e3c0d1e 100644 --- a/src/oplog/mod.rs +++ b/src/oplog/mod.rs @@ -1,11 +1,12 @@ -use crate::common::{BitfieldUpdate, Store, StoreInfo, StoreInfoInstruction}; -use crate::compact_encoding::{CompactEncoding, State}; -use crate::tree::MerkleTreeChangeset; -use crate::{Node, PartialKeypair}; use anyhow::{anyhow, Result}; use futures::future::Either; use std::convert::{TryFrom, TryInto}; +use crate::common::{BitfieldUpdate, Store, StoreInfo, StoreInfoInstruction}; +use crate::encoding::{CompactEncoding, HypercoreState}; +use crate::tree::MerkleTreeChangeset; +use crate::{Node, PartialKeypair}; + mod entry; mod header; @@ -72,7 +73,7 @@ enum OplogSlot { #[derive(Debug)] struct ValidateLeaderOutcome { - state: State, + state: HypercoreState, header_bit: bool, partial_bit: bool, } @@ -105,14 +106,14 @@ impl Oplog { if let Some(mut h2_outcome) = h2_outcome { let header_bits = [h1_outcome.header_bit, h2_outcome.header_bit]; let header: Header = if header_bits[0] == header_bits[1] { - h1_outcome.state.decode(&existing) + (*h1_outcome.state).decode(&existing) } else { - h2_outcome.state.decode(&existing) + (*h2_outcome.state).decode(&existing) }; (header, header_bits) } else { ( - h1_outcome.state.decode(&existing), + (*h1_outcome.state).decode(&existing), [h1_outcome.header_bit, h1_outcome.header_bit], ) }; @@ -131,7 +132,11 @@ impl Oplog { entries_length: 0, entries_byte_length: 0, }; - OplogOpenOutcome::new(oplog, h2_outcome.state.decode(&existing), Box::new([])) + OplogOpenOutcome::new( + oplog, + (*h2_outcome.state).decode(&existing), + Box::new([]), + ) } else if let Some(key_pair) = key_pair { // There is nothing in the oplog, start from new given key pair. Self::new(key_pair.clone()) @@ -152,7 +157,7 @@ impl Oplog { let entry: Entry = entry_outcome.state.decode(&existing); entries.push(entry); partials.push(entry_outcome.partial_bit); - entry_offset = entry_outcome.state.end; + entry_offset = (*entry_outcome.state).end; } else { break; } @@ -247,7 +252,7 @@ impl Oplog { let len = batch.len(); let header_bit = self.get_current_header_bit(); // Leave room for leaders - let mut state = State::new_with_start_and_end(0, len * 8); + let mut state = HypercoreState::new_with_start_and_end(0, len * 8); for entry in batch.iter() { state.preencode(entry); @@ -256,7 +261,7 @@ impl Oplog { let mut buffer = state.create_buffer(); for i in 0..len { let entry = &batch[i]; - state.start += 8; + (*state).start += 8; let start = state.start; let partial_bit: bool = atomic && i < len - 1; state.encode(entry, &mut buffer); @@ -303,7 +308,7 @@ impl Oplog { ) -> ([bool; 2], Box<[StoreInfo]>) { // The first 8 bytes will be filled with `prepend_leader`. let data_start_index: usize = 8; - let mut state = State::new_with_start_and_end(data_start_index, data_start_index); + let mut state = HypercoreState::new_with_start_and_end(data_start_index, data_start_index); // Get the right slot and header bit let (oplog_slot, header_bit) = @@ -318,13 +323,13 @@ impl Oplog { } // Preencode the new header - state.preencode(header); + (*state).preencode(header); // Create a buffer for the needed data let mut buffer = state.create_buffer(); // Encode the header - state.encode(header, &mut buffer); + (*state).encode(header, &mut buffer); // Finally prepend the buffer's 8 first bytes with a CRC, len and right bits Self::prepend_leader( @@ -357,12 +362,12 @@ impl Oplog { len: usize, header_bit: bool, partial_bit: bool, - state: &mut State, + state: &mut HypercoreState, buffer: &mut Box<[u8]>, ) { // The 4 bytes right before start of data is the length in 8+8+8+6=30 bits. The 31st bit is // the partial bit and 32nd bit the header bit. - state.start = state.start - len - 4; + (*state).start = (*state).start - len - 4; let len_u32: u32 = len.try_into().unwrap(); let partial_bit: u32 = if partial_bit { 2 } else { 0 }; let header_bit: u32 = if header_bit { 1 } else { 0 }; @@ -382,7 +387,7 @@ impl Oplog { if buffer.len() < index + 8 { return Ok(None); } - let mut state = State::new_with_start_and_end(index, buffer.len()); + let mut state = HypercoreState::new_with_start_and_end(index, buffer.len()); let stored_checksum: u32 = state.decode_u32(buffer); let combined: u32 = state.decode_u32(buffer); let len = usize::try_from(combined >> 2) diff --git a/src/tree/merkle_tree.rs b/src/tree/merkle_tree.rs index 8d8f1423..ffaaf021 100644 --- a/src/tree/merkle_tree.rs +++ b/src/tree/merkle_tree.rs @@ -1,5 +1,6 @@ use anyhow::Result; use anyhow::{anyhow, ensure}; +use compact_encoding::State; use ed25519_dalek::Signature; use futures::future::Either; use intmap::IntMap; @@ -7,7 +8,6 @@ use std::convert::TryFrom; use crate::common::NodeByteRange; use crate::common::{Proof, ValuelessProof}; -use crate::compact_encoding::State; use crate::crypto::{Hash, PublicKey}; use crate::oplog::HeaderTree; use crate::{ diff --git a/tests/compact_encoding.rs b/tests/compact_encoding.rs deleted file mode 100644 index bcf10c15..00000000 --- a/tests/compact_encoding.rs +++ /dev/null @@ -1,185 +0,0 @@ -use hypercore::compact_encoding::{CompactEncoding, State}; - -// The max value for 1 byte length is 252 -const MAX_ONE_BYTE_UINT: u8 = 252; - -// The min value for 2 byte length is 253 -const MIN_TWO_BYTE_UINT: u8 = 253; - -#[test] -fn cenc_basic() { - let str_value_1 = "foo"; - let str_value_2 = (0..MAX_ONE_BYTE_UINT).map(|_| "X").collect::(); - let u32_value_3: u32 = u32::MAX; - let u32_value_4: u32 = 0xF0E1D2C3; - - let mut enc_state = State::new(); - enc_state.preencode_str(&str_value_1); - enc_state.preencode(&str_value_2); - enc_state.preencode(&u32_value_3); - enc_state.preencode(&u32_value_4); - let mut buffer = enc_state.create_buffer(); - // Strings: 1 byte for length, 3/252 bytes for content - // u32: 1 byte for u32 signifier, 4 bytes for data - assert_eq!(buffer.len(), 1 + 3 + 1 + 252 + 1 + 4 + 1 + 4); - enc_state.encode_str(&str_value_1, &mut buffer); - enc_state.encode(&str_value_2, &mut buffer); - enc_state.encode(&u32_value_3, &mut buffer); - enc_state.encode(&u32_value_4, &mut buffer); - let mut dec_state = State::from_buffer(&buffer); - let str_value_1_ret: String = dec_state.decode(&buffer); - assert_eq!(str_value_1, str_value_1_ret); - let str_value_2_ret: String = dec_state.decode(&buffer); - assert_eq!(str_value_2, str_value_2_ret); - let u32_value_3_ret: u32 = dec_state.decode(&buffer); - assert_eq!(u32_value_3, u32_value_3_ret); - let u32_value_4_ret: u32 = dec_state.decode(&buffer); - assert_eq!(u32_value_4, u32_value_4_ret); -} - -#[test] -fn cenc_string_long() { - let str_value = (0..MIN_TWO_BYTE_UINT).map(|_| "X").collect::(); - assert_eq!(str_value.len(), 253); - let mut enc_state = State::new(); - enc_state.preencode(&str_value); - let mut buffer = enc_state.create_buffer(); - // 1 byte for u16 signifier, 2 bytes for length, 256 bytes for data - assert_eq!(buffer.len(), 1 + 2 + 253); - enc_state.encode(&str_value, &mut buffer); - let mut dec_state = State::from_buffer(&buffer); - let str_value_ret: String = dec_state.decode(&buffer); - assert_eq!(str_value, str_value_ret); -} - -#[test] -fn cenc_u32_as_u16() { - let u32_value: u32 = u16::MAX.into(); - let mut enc_state = State::new(); - enc_state.preencode(&u32_value); - let mut buffer = enc_state.create_buffer(); - // 1 byte for u16 signifier, 2 bytes for length - assert_eq!(buffer.len(), 1 + 2); - enc_state.encode(&u32_value, &mut buffer); - let mut dec_state = State::from_buffer(&buffer); - let u32_value_ret: u32 = dec_state.decode(&buffer); - assert_eq!(u32_value, u32_value_ret); -} - -#[test] -fn cenc_u32_as_u8() { - let u32_value: u32 = MAX_ONE_BYTE_UINT.into(); - let mut enc_state = State::new(); - enc_state.preencode(&u32_value); - let mut buffer = enc_state.create_buffer(); - // 1 byte for data - assert_eq!(buffer.len(), 1); - enc_state.encode(&u32_value, &mut buffer); - let mut dec_state = State::from_buffer(&buffer); - let u32_value_ret: u32 = dec_state.decode(&buffer); - assert_eq!(u32_value, u32_value_ret); -} - -#[test] -fn cenc_u64() { - let u64_value: u64 = 0xF0E1D2C3B4A59687; - let mut enc_state = State::new(); - enc_state.preencode(&u64_value); - let mut buffer = enc_state.create_buffer(); - // 1 byte for u64 signifier, 8 bytes for length - assert_eq!(buffer.len(), 1 + 8); - enc_state.encode(&u64_value, &mut buffer); - let mut dec_state = State::from_buffer(&buffer); - let u64_value_ret: u64 = dec_state.decode(&buffer); - assert_eq!(u64_value, u64_value_ret); -} - -#[test] -fn cenc_u64_as_u32() { - let u64_value: u64 = u32::MAX.into(); - let mut enc_state = State::new(); - enc_state.preencode(&u64_value); - let mut buffer = enc_state.create_buffer(); - // 1 byte for u32 signifier, 4 bytes for length - assert_eq!(buffer.len(), 1 + 4); - enc_state.encode(&u64_value, &mut buffer); - let mut dec_state = State::from_buffer(&buffer); - let u64_value_ret: u64 = dec_state.decode(&buffer); - assert_eq!(u64_value, u64_value_ret); -} - -#[test] -fn cenc_buffer() { - let buf_value_1 = vec![0xFF, 0x00].into_boxed_slice(); - let buf_value_2 = vec![0xEE, 0x11, 0x22].into_boxed_slice(); - let mut enc_state = State::new(); - enc_state.preencode(&buf_value_1); - enc_state.preencode(&buf_value_2); - let mut buffer = enc_state.create_buffer(); - // 1 byte for length, 2 bytes for data - // 1 byte for length, 3 bytes for data - assert_eq!(buffer.len(), 1 + 2 + 1 + 3); - enc_state.encode(&buf_value_1, &mut buffer); - enc_state.encode(&buf_value_2, &mut buffer); - let mut dec_state = State::from_buffer(&buffer); - let buf_value_1_ret: Box<[u8]> = dec_state.decode(&buffer); - let buf_value_2_ret: Box<[u8]> = dec_state.decode(&buffer); - assert_eq!(buf_value_1, buf_value_1_ret); - assert_eq!(buf_value_2, buf_value_2_ret); -} - -#[test] -fn cenc_vec() { - let buf_value_1: Vec = vec![0xFF, 0x00]; - let buf_value_2: Vec = vec![0xFFFFFFFF, 0x11223344, 0x99887766]; - let mut enc_state = State::new(); - enc_state.preencode(&buf_value_1); - enc_state.preencode(&buf_value_2); - let mut buffer = enc_state.create_buffer(); - // 1 byte for length, 2 bytes for data - // 1 byte for length, 4*3 bytes for data - assert_eq!(buffer.len(), 1 + 2 + 1 + 12); - enc_state.encode(&buf_value_1, &mut buffer); - enc_state.encode(&buf_value_2, &mut buffer); - let mut dec_state = State::from_buffer(&buffer); - let buf_value_1_ret: Vec = dec_state.decode(&buffer); - let buf_value_2_ret: Vec = dec_state.decode(&buffer); - assert_eq!(buf_value_1, buf_value_1_ret); - assert_eq!(buf_value_2, buf_value_2_ret); -} - -#[test] -fn cenc_string_array() { - let string_array_value = vec!["first".to_string(), "second".to_string()]; - let mut enc_state = State::new(); - enc_state.preencode(&string_array_value); - let mut buffer = enc_state.create_buffer(); - // 1 byte for array length, - // 1 byte for string length, 5 bytes for string, - // 1 byte for string length, 6 bytes for string - assert_eq!(buffer.len(), 1 + 1 + 5 + 1 + 6); - enc_state.encode(&string_array_value, &mut buffer); - let mut dec_state = State::from_buffer(&buffer); - let string_array_value_ret: Vec = dec_state.decode(&buffer); - assert_eq!(string_array_value, string_array_value_ret); -} - -#[test] -fn cenc_fixed_and_raw() { - let buf_value_1: Vec = vec![0xFF; 32]; - let buf_value_2: Vec = vec![0xFF, 0x11, 0x99]; - let mut enc_state = State::new(); - enc_state.preencode_fixed_32(); - enc_state.preencode_raw_buffer(&buf_value_2); - let mut buffer = enc_state.create_buffer(); - // 32 bytes for data - // 3 bytes for data - assert_eq!(buffer.len(), 32 + 3); - enc_state.encode_fixed_32(&buf_value_1, &mut buffer); - enc_state.encode_raw_buffer(&buf_value_2, &mut buffer); - let mut dec_state = State::from_buffer(&buffer); - let buf_value_1_ret: Vec = dec_state.decode_fixed_32(&buffer).to_vec(); - let buf_value_2_ret: Vec = dec_state.decode_raw_buffer(&buffer); - assert_eq!(buf_value_1, buf_value_1_ret); - assert_eq!(buf_value_2, buf_value_2_ret); -} From ea6fed02ef091a0374219eb75fe69dd316e16f10 Mon Sep 17 00:00:00 2001 From: Timo Tiuraniemi Date: Wed, 22 Mar 2023 11:35:04 +0200 Subject: [PATCH 099/157] Use explicit branch main --- Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index d3d7191c..01fc0638 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -26,7 +26,7 @@ blake2-rfc = "0.2.18" byteorder = "1.3.4" ed25519-dalek = "1.0.1" anyhow = "1.0.26" -compact-encoding = { git = "https://github.com/datrs/compact-encoding" } +compact-encoding = { git = "https://github.com/datrs/compact-encoding", branch = "main" } flat-tree = { git = "https://github.com/ttiurani/flat-tree", branch = "v10" } lazy_static = "1.4.0" memory-pager = "0.9.0" From 02befa67759bfdffaed514365f85541d99298d4b Mon Sep 17 00:00:00 2001 From: Timo Tiuraniemi Date: Thu, 23 Mar 2023 09:28:39 +0200 Subject: [PATCH 100/157] Remove v9-only file --- src/crypto/root.rs | 52 ---------------------------------------------- 1 file changed, 52 deletions(-) delete mode 100644 src/crypto/root.rs diff --git a/src/crypto/root.rs b/src/crypto/root.rs deleted file mode 100644 index 12a6713d..00000000 --- a/src/crypto/root.rs +++ /dev/null @@ -1,52 +0,0 @@ -//! Root node type. Functions as an intermediate type for hash methods that -//! operate on Root. -//! -//! ## Why? -//! Both `merkle-tree-stream` and `hypercore` have `Node` types. Even if in most -//! cases these types don't overlap, in a select few cases both need to be -//! passed to the same function. So in order to facilitate that, the `Root` type -//! is created. It's entirely borrowed, and allows passing either type down into -//! a function that accepts `Root`. - -/// Root node found in flat-tree. -pub struct Root<'a> { - index: &'a u64, - length: &'a u64, - hash: &'a [u8], -} - -impl<'a> Root<'a> { - /// Create a new instance. - #[inline] - pub fn new(index: &'a u64, length: &'a u64, hash: &'a [u8]) -> Self { - Self { - index, - length, - hash, - } - } - - /// Get the index at which this root was found inside a `flat-tree`. - #[inline] - pub fn index(&self) -> &u64 { - &self.index - } - - /// Get the lenght of the data. - #[inline] - pub fn len(&self) -> &u64 { - &self.length - } - - /// Check if the content is empty. - #[inline] - pub fn is_empty(&self) -> bool { - *self.length == 0 - } - - /// Get the hash. - #[inline] - pub fn hash(&self) -> &'a [u8] { - &self.hash - } -} From 0dcf552efc0ff7e1fdf53fb0198273045ec629b0 Mon Sep 17 00:00:00 2001 From: Timo Tiuraniemi Date: Thu, 23 Mar 2023 10:22:46 +0200 Subject: [PATCH 101/157] 1/n switch from anyhow to thiserror --- Cargo.toml | 1 + src/common/error.rs | 28 ++++++++++++++++++++++++++++ src/common/mod.rs | 2 ++ src/crypto/key_pair.rs | 21 +++++++++++++-------- src/lib.rs | 5 ++--- 5 files changed, 46 insertions(+), 11 deletions(-) create mode 100644 src/common/error.rs diff --git a/Cargo.toml b/Cargo.toml index 01fc0638..6fcee19e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -25,6 +25,7 @@ edition = "2018" blake2-rfc = "0.2.18" byteorder = "1.3.4" ed25519-dalek = "1.0.1" +thiserror = "1" anyhow = "1.0.26" compact-encoding = { git = "https://github.com/datrs/compact-encoding", branch = "main" } flat-tree = { git = "https://github.com/ttiurani/flat-tree", branch = "v10" } diff --git a/src/common/error.rs b/src/common/error.rs new file mode 100644 index 00000000..ee06d5cc --- /dev/null +++ b/src/common/error.rs @@ -0,0 +1,28 @@ +use thiserror::Error; + +/// Common error type for the hypercore interface +#[derive(Error, Debug)] +pub enum HypercoreError { + /// Invalid signature + #[error("Given signature was invalid.")] + InvalidSignature, + /// Unexpected IO error occured + #[error("Unrecoverable input/output error occured.{}", + .context.as_ref().map_or_else(String::new, |ctx| format!(" Context: {}.", ctx)))] + IO { + /// Context for the error + context: Option, + /// Original source error + #[source] + source: std::io::Error, + }, +} + +impl From for HypercoreError { + fn from(err: std::io::Error) -> Self { + Self::IO { + context: None, + source: err, + } + } +} diff --git a/src/common/mod.rs b/src/common/mod.rs index d3b63c12..e13812ca 100644 --- a/src/common/mod.rs +++ b/src/common/mod.rs @@ -1,7 +1,9 @@ +mod error; mod node; mod peer; mod store; +pub use self::error::HypercoreError; pub use self::node::{Node, NodeByteRange}; pub use self::peer::{ DataBlock, DataHash, DataSeek, DataUpgrade, Proof, RequestBlock, RequestSeek, RequestUpgrade, diff --git a/src/crypto/key_pair.rs b/src/crypto/key_pair.rs index 53503afb..0a22e1e3 100644 --- a/src/crypto/key_pair.rs +++ b/src/crypto/key_pair.rs @@ -2,10 +2,11 @@ pub use ed25519_dalek::{ExpandedSecretKey, Keypair, PublicKey, SecretKey, Signature, Verifier}; -use anyhow::{bail, ensure, Result}; use rand::rngs::{OsRng, StdRng}; use rand::SeedableRng; +use crate::HypercoreError; + /// Generate a new `Ed25519` key pair. pub fn generate() -> Keypair { let mut rng = StdRng::from_rng(OsRng::default()).unwrap(); @@ -18,15 +19,19 @@ pub fn sign(public_key: &PublicKey, secret: &SecretKey, msg: &[u8]) -> Signature } /// Verify a signature on a message with a keypair's public key. -pub fn verify(public: &PublicKey, msg: &[u8], sig: Option<&Signature>) -> Result<()> { +pub fn verify( + public: &PublicKey, + msg: &[u8], + sig: Option<&Signature>, +) -> Result<(), HypercoreError> { match sig { - None => bail!("Signature verification failed"), + None => Err(HypercoreError::InvalidSignature), Some(sig) => { - ensure!( - public.verify(msg, sig).is_ok(), - "Signature verification failed" - ); - Ok(()) + if public.verify(msg, sig).is_ok() { + Ok(()) + } else { + Err(HypercoreError::InvalidSignature) + } } } } diff --git a/src/lib.rs b/src/lib.rs index 96e0ab87..5a080d61 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -38,10 +38,9 @@ mod oplog; mod storage; mod tree; -pub use crate::common::Node; pub use crate::common::{ - DataBlock, DataHash, DataSeek, DataUpgrade, Proof, RequestBlock, RequestSeek, RequestUpgrade, - Store, + DataBlock, DataHash, DataSeek, DataUpgrade, HypercoreError, Node, Proof, RequestBlock, + RequestSeek, RequestUpgrade, Store, }; pub use crate::core::Hypercore; pub use crate::crypto::{generate_keypair, sign, verify, Signature}; From 13203018d92e86f8dbb96391329ac2b21c56c0c6 Mon Sep 17 00:00:00 2001 From: Timo Tiuraniemi Date: Thu, 23 Mar 2023 10:25:43 +0200 Subject: [PATCH 102/157] Remove v9 file --- src/event.rs | 3 --- src/lib.rs | 2 -- 2 files changed, 5 deletions(-) delete mode 100644 src/event.rs diff --git a/src/event.rs b/src/event.rs deleted file mode 100644 index 37d072b7..00000000 --- a/src/event.rs +++ /dev/null @@ -1,3 +0,0 @@ -/// Events emitted. -#[derive(Debug, Clone, PartialEq)] -pub enum Event {} diff --git a/src/lib.rs b/src/lib.rs index 5a080d61..f539fe45 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -33,7 +33,6 @@ mod common; mod core; mod crypto; mod data; -mod event; mod oplog; mod storage; mod tree; @@ -44,7 +43,6 @@ pub use crate::common::{ }; pub use crate::core::Hypercore; pub use crate::crypto::{generate_keypair, sign, verify, Signature}; -pub use crate::event::Event; pub use crate::storage::{PartialKeypair, Storage}; pub use ed25519_dalek::{ ExpandedSecretKey, Keypair, PublicKey, SecretKey, EXPANDED_SECRET_KEY_LENGTH, KEYPAIR_LENGTH, From ba7987ffbbe4b4020b3df92a7dfb11bb637bc01b Mon Sep 17 00:00:00 2001 From: Timo Tiuraniemi Date: Thu, 23 Mar 2023 12:32:24 +0200 Subject: [PATCH 103/157] 2/n switch from anyhow to thiserror --- src/common/error.rs | 6 ++++++ src/oplog/mod.rs | 14 ++++++++------ 2 files changed, 14 insertions(+), 6 deletions(-) diff --git a/src/common/error.rs b/src/common/error.rs index ee06d5cc..52df960e 100644 --- a/src/common/error.rs +++ b/src/common/error.rs @@ -6,6 +6,12 @@ pub enum HypercoreError { /// Invalid signature #[error("Given signature was invalid.")] InvalidSignature, + /// Invalid checksum + #[error("Invalid checksum.")] + InvalidChecksum, + /// Empty storage + #[error("Empty storage.")] + EmptyStorage, /// Unexpected IO error occured #[error("Unrecoverable input/output error occured.{}", .context.as_ref().map_or_else(String::new, |ctx| format!(" Context: {}.", ctx)))] diff --git a/src/oplog/mod.rs b/src/oplog/mod.rs index 3e3c0d1e..473f234b 100644 --- a/src/oplog/mod.rs +++ b/src/oplog/mod.rs @@ -1,11 +1,10 @@ -use anyhow::{anyhow, Result}; use futures::future::Either; use std::convert::{TryFrom, TryInto}; use crate::common::{BitfieldUpdate, Store, StoreInfo, StoreInfoInstruction}; use crate::encoding::{CompactEncoding, HypercoreState}; use crate::tree::MerkleTreeChangeset; -use crate::{Node, PartialKeypair}; +use crate::{HypercoreError, Node, PartialKeypair}; mod entry; mod header; @@ -87,7 +86,7 @@ impl Oplog { pub fn open( key_pair: &Option, info: Option, - ) -> Result> { + ) -> Result, HypercoreError> { match info { None => Ok(Either::Left(StoreInfoInstruction::new_all_content( Store::Oplog, @@ -142,7 +141,7 @@ impl Oplog { Self::new(key_pair.clone()) } else { // The storage is empty and no key pair given, erroring - return Err(anyhow!("Nothing stored in oplog and key pair not given")); + return Err(HypercoreError::EmptyStorage); }; // Read headers that might be stored in the existing content @@ -383,7 +382,10 @@ impl Oplog { /// Validates that leader at given index is valid, and returns header and partial bits and /// `State` for the header/entry that the leader was for. - fn validate_leader(index: usize, buffer: &Box<[u8]>) -> Result> { + fn validate_leader( + index: usize, + buffer: &Box<[u8]>, + ) -> Result, HypercoreError> { if buffer.len() < index + 8 { return Ok(None); } @@ -406,7 +408,7 @@ impl Oplog { state.end = state.start + len; let calculated_checksum = crc32fast::hash(&buffer[index + 4..state.end]); if calculated_checksum != stored_checksum { - return Err(anyhow!("Checksums do not match")); + return Err(HypercoreError::InvalidChecksum); }; Ok(Some(ValidateLeaderOutcome { From 92046ab7848c4a59d7e354d11b2216df7ff65236 Mon Sep 17 00:00:00 2001 From: Timo Tiuraniemi Date: Thu, 23 Mar 2023 14:24:59 +0200 Subject: [PATCH 104/157] 3/n switch from anyhow to thiserror --- src/common/error.rs | 46 ++++++-- src/common/store.rs | 11 ++ src/crypto/key_pair.rs | 8 +- src/oplog/mod.rs | 8 +- src/tree/merkle_tree.rs | 177 ++++++++++++++++++------------ src/tree/merkle_tree_changeset.rs | 9 +- 6 files changed, 176 insertions(+), 83 deletions(-) diff --git a/src/common/error.rs b/src/common/error.rs index 52df960e..fc11458d 100644 --- a/src/common/error.rs +++ b/src/common/error.rs @@ -1,20 +1,52 @@ use thiserror::Error; +use crate::Store; + /// Common error type for the hypercore interface #[derive(Error, Debug)] pub enum HypercoreError { + /// Bad argument + #[error("Bad argument. {context}")] + BadArgument { + /// Context for the error + context: String, + }, /// Invalid signature - #[error("Given signature was invalid.")] - InvalidSignature, + #[error("Given signature was invalid. {context}")] + InvalidSignature { + /// Context for the error + context: String, + }, /// Invalid checksum - #[error("Invalid checksum.")] - InvalidChecksum, + #[error("Invalid checksum. {context}")] + InvalidChecksum { + /// Context for the error + context: String, + }, /// Empty storage - #[error("Empty storage.")] - EmptyStorage, + #[error("Empty storage: {store}.")] + EmptyStorage { + /// Store that was found empty + store: Store, + }, + /// Corrupt storage + #[error("Corrupt storage: {store}.{}", + .context.as_ref().map_or_else(String::new, |ctx| format!(" Context: {}.", ctx)))] + CorruptStorage { + /// Store that was corrupt + store: Store, + /// Context for the error + context: Option, + }, + /// Invalid operation + #[error("Invalid operation. {context}")] + InvalidOperation { + /// Context for the error + context: String, + }, /// Unexpected IO error occured #[error("Unrecoverable input/output error occured.{}", - .context.as_ref().map_or_else(String::new, |ctx| format!(" Context: {}.", ctx)))] + .context.as_ref().map_or_else(String::new, |ctx| format!(" {}.", ctx)))] IO { /// Context for the error context: Option, diff --git a/src/common/store.rs b/src/common/store.rs index 8d3f332c..fa1a7d68 100644 --- a/src/common/store.rs +++ b/src/common/store.rs @@ -11,6 +11,17 @@ pub enum Store { Oplog, } +impl std::fmt::Display for Store { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Store::Tree => write!(f, "tree"), + Store::Data => write!(f, "data"), + Store::Bitfield => write!(f, "bitfield"), + Store::Oplog => write!(f, "oplog"), + } + } +} + /// Information type about a store. #[derive(Debug, PartialEq)] pub enum StoreInfoType { diff --git a/src/crypto/key_pair.rs b/src/crypto/key_pair.rs index 0a22e1e3..ebafcdc8 100644 --- a/src/crypto/key_pair.rs +++ b/src/crypto/key_pair.rs @@ -25,12 +25,16 @@ pub fn verify( sig: Option<&Signature>, ) -> Result<(), HypercoreError> { match sig { - None => Err(HypercoreError::InvalidSignature), + None => Err(HypercoreError::InvalidSignature { + context: "No signature provided.".to_string(), + }), Some(sig) => { if public.verify(msg, sig).is_ok() { Ok(()) } else { - Err(HypercoreError::InvalidSignature) + Err(HypercoreError::InvalidSignature { + context: "Signature could not be verified.".to_string(), + }) } } } diff --git a/src/oplog/mod.rs b/src/oplog/mod.rs index 473f234b..1b77393e 100644 --- a/src/oplog/mod.rs +++ b/src/oplog/mod.rs @@ -141,7 +141,9 @@ impl Oplog { Self::new(key_pair.clone()) } else { // The storage is empty and no key pair given, erroring - return Err(HypercoreError::EmptyStorage); + return Err(HypercoreError::EmptyStorage { + store: Store::Oplog, + }); }; // Read headers that might be stored in the existing content @@ -408,7 +410,9 @@ impl Oplog { state.end = state.start + len; let calculated_checksum = crc32fast::hash(&buffer[index + 4..state.end]); if calculated_checksum != stored_checksum { - return Err(HypercoreError::InvalidChecksum); + return Err(HypercoreError::InvalidChecksum { + context: "Calculated signature does not match oplog signature".to_string(), + }); }; Ok(Some(ValidateLeaderOutcome { diff --git a/src/tree/merkle_tree.rs b/src/tree/merkle_tree.rs index ffaaf021..17342c2f 100644 --- a/src/tree/merkle_tree.rs +++ b/src/tree/merkle_tree.rs @@ -1,13 +1,10 @@ -use anyhow::Result; -use anyhow::{anyhow, ensure}; use compact_encoding::State; use ed25519_dalek::Signature; use futures::future::Either; use intmap::IntMap; use std::convert::TryFrom; -use crate::common::NodeByteRange; -use crate::common::{Proof, ValuelessProof}; +use crate::common::{HypercoreError, NodeByteRange, Proof, ValuelessProof}; use crate::crypto::{Hash, PublicKey}; use crate::oplog::HeaderTree; use crate::{ @@ -41,7 +38,7 @@ impl MerkleTree { pub fn open( header_tree: &HeaderTree, infos: Option<&[StoreInfo]>, - ) -> Result, Self>> { + ) -> Result, Self>, HypercoreError> { match infos { None => { let root_indices = get_root_indices(&header_tree.length); @@ -69,10 +66,14 @@ impl MerkleTree { for i in 0..root_indices.len() { let index = root_indices[i]; - ensure!( - index == index_from_info(&infos[i]), - "Given slices vector not in the correct order" - ); + if index != index_from_info(&infos[i]) { + return Err(HypercoreError::CorruptStorage { + store: Store::Tree, + context: Some( + "Given slices vector not in the correct order".to_string(), + ), + }); + } let data = infos[i].data.as_ref().unwrap(); let node = node_from_bytes(&index, data); byte_length += node.length; @@ -85,7 +86,11 @@ impl MerkleTree { length = length / 2; } let signature: Option = if header_tree.signature.len() > 0 { - Some(Signature::try_from(&*header_tree.signature)?) + Some(Signature::try_from(&*header_tree.signature).map_err(|err| { + HypercoreError::InvalidSignature { + context: "Could not parse signature".to_string(), + } + })?) } else { None }; @@ -112,11 +117,11 @@ impl MerkleTree { } /// Commit a created changeset to the tree. - pub fn commit(&mut self, changeset: MerkleTreeChangeset) -> Result<()> { + pub fn commit(&mut self, changeset: MerkleTreeChangeset) -> Result<(), HypercoreError> { if !self.commitable(&changeset) { - return Err(anyhow!( - "Tree was modified during changeset, refusing to commit" - )); + return Err(HypercoreError::InvalidOperation { + context: "Tree was modified during changeset, refusing to commit".to_string(), + }); } if changeset.upgraded { @@ -151,7 +156,7 @@ impl MerkleTree { &self, hypercore_index: u64, infos: Option<&[StoreInfo]>, - ) -> Result, NodeByteRange>> { + ) -> Result, NodeByteRange>, HypercoreError> { let index = self.validate_hypercore_index(hypercore_index)?; // Get nodes out of incoming infos let nodes: IntMap> = infos_to_nodes(infos); @@ -199,7 +204,7 @@ impl MerkleTree { &self, hypercore_index: u64, infos: Option<&[StoreInfo]>, - ) -> Result, u64>> { + ) -> Result, u64>, HypercoreError> { let index = self.validate_hypercore_index(hypercore_index)?; self.byte_offset_from_index(index, infos) } @@ -210,7 +215,7 @@ impl MerkleTree { hypercore_index: u64, changeset: &MerkleTreeChangeset, infos: Option<&[StoreInfo]>, - ) -> Result, u64>> { + ) -> Result, u64>, HypercoreError> { if self.length == hypercore_index { return Ok(Either::Right(self.byte_length)); } @@ -263,7 +268,7 @@ impl MerkleTree { length: u64, fork: u64, infos: Option<&[StoreInfo]>, - ) -> Result, MerkleTreeChangeset>> { + ) -> Result, MerkleTreeChangeset>, HypercoreError> { let head = length * 2; let mut full_roots = vec![]; flat_tree::full_roots(head, &mut full_roots); @@ -320,7 +325,7 @@ impl MerkleTree { seek: Option<&RequestSeek>, upgrade: Option<&RequestUpgrade>, infos: Option<&[StoreInfo]>, - ) -> Result, ValuelessProof>> { + ) -> Result, ValuelessProof>, HypercoreError> { let nodes: IntMap> = infos_to_nodes(infos); let mut instructions: Vec = Vec::new(); let fork = self.fork; @@ -335,7 +340,9 @@ impl MerkleTree { let indexed = normalize_indexed(block, hash); if from >= to || to > head { - return Err(anyhow!("Invalid upgrade")); + return Err(HypercoreError::InvalidOperation { + context: "Invalid upgrade".to_string(), + }); } let mut sub_tree = head; @@ -349,9 +356,10 @@ impl MerkleTree { let mut untrusted_sub_tree = false; if let Some(indexed) = indexed.as_ref() { if seek.is_some() && upgrade.is_some() && indexed.index >= from { - return Err(anyhow!( - "Cannot both do a seek and block/hash request when upgrading" - )); + return Err(HypercoreError::InvalidOperation { + context: "Cannot both do a seek and block/hash request when upgrading" + .to_string(), + }); } if let Some(upgrade) = upgrade.as_ref() { @@ -496,7 +504,7 @@ impl MerkleTree { proof: &Proof, public_key: &PublicKey, infos: Option<&[StoreInfo]>, - ) -> Result, MerkleTreeChangeset>> { + ) -> Result, MerkleTreeChangeset>, HypercoreError> { let nodes: IntMap> = infos_to_nodes(infos); let mut instructions: Vec = Vec::new(); let mut changeset = self.changeset(); @@ -528,10 +536,13 @@ impl MerkleTree { } Either::Right(verified_block_root_node) => { if verified_block_root_node.hash != unverified_block_root_node.hash { - return Err(anyhow!( - "Invalid checksum at node {}", - unverified_block_root_node.index - )); + return Err(HypercoreError::InvalidChecksum { + context: format!( + "Invalid checksum at node {}, store {}", + unverified_block_root_node.index, + Store::Tree + ), + }); } } } @@ -549,7 +560,7 @@ impl MerkleTree { &self, index: u64, infos: Option<&[StoreInfo]>, - ) -> Result, u64>> { + ) -> Result, u64>, HypercoreError> { let head = 2 * self.length; let mut iter = flat_tree::Iterator::new(index); let iter_right_span = iter.index() + iter.factor() / 2 - 1; @@ -653,7 +664,7 @@ impl MerkleTree { } /// Validates given hypercore index and returns tree index - fn validate_hypercore_index(&self, hypercore_index: u64) -> Result { + fn validate_hypercore_index(&self, hypercore_index: u64) -> Result { // Converts a hypercore index into a merkle tree index let index = hypercore_index_into_merkle_tree_index(hypercore_index); @@ -665,10 +676,9 @@ impl MerkleTree { flat_tree::right_span(index) }; if compare_index >= head { - return Err(anyhow!( - "Hypercore index {} is out of bounds", - hypercore_index - )); + return Err(HypercoreError::BadArgument { + context: format!("Hypercore index {} is out of bounds", hypercore_index), + }); } Ok(index) } @@ -677,7 +687,7 @@ impl MerkleTree { &self, index: u64, infos: Option<&[StoreInfo]>, - ) -> Result, u64>> { + ) -> Result, u64>, HypercoreError> { // Get nodes out of incoming infos let nodes: IntMap> = infos_to_nodes(infos); // Get offset @@ -695,7 +705,7 @@ impl MerkleTree { &self, index: u64, nodes: &IntMap>, - ) -> Result, u64>> { + ) -> Result, u64>, HypercoreError> { let index = if (index & 1) == 1 { flat_tree::left_span(index) } else { @@ -738,24 +748,25 @@ impl MerkleTree { }; } - Err(anyhow!( - "Could not calculate byte offset for index {}", - index - )) + Err(HypercoreError::BadArgument { + context: format!("Could not calculate byte offset for index {}", index), + }) } fn required_node( &self, index: u64, nodes: &IntMap>, - ) -> Result> { + ) -> Result, HypercoreError> { match self.node(index, nodes, false)? { Either::Left(value) => Ok(Either::Left(value)), Either::Right(node) => { if let Some(node) = node { Ok(Either::Right(node)) } else { - Err(anyhow!("Node at {} was required", index)) + Err(HypercoreError::InvalidOperation { + context: format!("Node at {} is required, store {}", index, Store::Tree), + }) } } } @@ -765,7 +776,7 @@ impl MerkleTree { &self, index: u64, nodes: &IntMap>, - ) -> Result>> { + ) -> Result>, HypercoreError> { self.node(index, nodes, true) } @@ -774,7 +785,7 @@ impl MerkleTree { index: u64, nodes: &IntMap>, allow_miss: bool, - ) -> Result>> { + ) -> Result>, HypercoreError> { // First check if unflushed already has the node if let Some(node) = self.unflushed.get(index) { if node.blank || (self.truncated && node.index >= 2 * self.truncate_to) { @@ -782,7 +793,13 @@ impl MerkleTree { return if allow_miss { Ok(Either::Right(None)) } else { - Err(anyhow!("Could not load node: {}", index)) + Err(HypercoreError::InvalidOperation { + context: format!( + "Could not load node: {}, store {}, unflushed", + index, + Store::Tree + ), + }) }; } return Ok(Either::Right(Some(node.clone()))); @@ -796,14 +813,26 @@ impl MerkleTree { return if allow_miss { Ok(Either::Right(None)) } else { - Err(anyhow!("Could not load node: {}", index)) + Err(HypercoreError::InvalidOperation { + context: format!( + "Could not load node: {}, store {}, blank", + index, + Store::Tree + ), + }) }; } return Ok(Either::Right(Some(node.clone()))); } else if allow_miss { return Ok(Either::Right(None)); } else { - return Err(anyhow!("Could not load node: {}", index)); + return Err(HypercoreError::InvalidOperation { + context: format!( + "Could not load node: {}, store {}, empty", + index, + Store::Tree + ), + }); } } @@ -823,7 +852,7 @@ impl MerkleTree { head: u64, bytes: u64, nodes: &IntMap>, - ) -> Result, u64>> { + ) -> Result, u64>, HypercoreError> { let mut instructions: Vec = Vec::new(); let mut roots = vec![]; flat_tree::full_roots(head, &mut roots); @@ -869,7 +898,7 @@ impl MerkleTree { root: u64, bytes: u64, nodes: &IntMap>, - ) -> Result, u64>> { + ) -> Result, u64>, HypercoreError> { if bytes == 0 { return Ok(Either::Right(root)); } @@ -917,7 +946,7 @@ impl MerkleTree { root: u64, bytes: u64, nodes: &IntMap>, - ) -> Result, u64>> { + ) -> Result, u64>, HypercoreError> { let mut instructions: Vec = Vec::new(); let offset_or_instructions = self.byte_offset_from_nodes(root, nodes)?; let mut bytes = bytes; @@ -927,7 +956,9 @@ impl MerkleTree { } Either::Right(offset) => { if offset > bytes { - return Err(anyhow!("Invalid seek")); + return Err(HypercoreError::InvalidOperation { + context: "Invalid seek, wrong offset".to_string(), + }); } if offset == bytes { return Ok(Either::Right(root)); @@ -940,7 +971,9 @@ impl MerkleTree { } Either::Right(node) => { if node.length <= bytes { - return Err(anyhow!("Invalid seek")); + return Err(HypercoreError::InvalidOperation { + context: "Invalid seek, wrong length".to_string(), + }); } } } @@ -964,7 +997,7 @@ impl MerkleTree { root: u64, p: &mut LocalProof, nodes: &IntMap>, - ) -> Result, ()>> { + ) -> Result, ()>, HypercoreError> { if let Some(indexed) = indexed { let mut iter = flat_tree::Iterator::new(indexed.index); let mut instructions: Vec = Vec::new(); @@ -1023,7 +1056,7 @@ impl MerkleTree { root: u64, p: &mut LocalProof, nodes: &IntMap>, - ) -> Result, ()>> { + ) -> Result, ()>, HypercoreError> { let mut iter = flat_tree::Iterator::new(seek_root); let mut instructions: Vec = Vec::new(); let mut seek_nodes: Vec = Vec::new(); @@ -1067,7 +1100,7 @@ impl MerkleTree { sub_tree: u64, p: &mut LocalProof, nodes: &IntMap>, - ) -> Result, ()>> { + ) -> Result, ()>, HypercoreError> { let mut instructions: Vec = Vec::new(); let mut upgrade: Vec = Vec::new(); let mut has_upgrade = false; @@ -1174,7 +1207,7 @@ impl MerkleTree { to: u64, p: &mut LocalProof, nodes: &IntMap>, - ) -> Result, ()>> { + ) -> Result, ()>, HypercoreError> { let mut instructions: Vec = Vec::new(); let mut additional_upgrade: Vec = Vec::new(); let mut has_additional_upgrade = false; @@ -1262,7 +1295,7 @@ fn verify_tree( hash: Option<&DataHash>, seek: Option<&DataSeek>, changeset: &mut MerkleTreeChangeset, -) -> Result> { +) -> Result, HypercoreError> { let untrusted_node: Option = normalize_data(block, hash); if untrusted_node.is_none() { @@ -1325,7 +1358,7 @@ fn verify_upgrade( block_root: Option<&Node>, public_key: &PublicKey, changeset: &mut MerkleTreeChangeset, -) -> Result { +) -> Result { let mut q = if let Some(block_root) = block_root { NodeQueue::new(upgrade.nodes.clone(), Some(block_root.clone())) } else { @@ -1371,7 +1404,9 @@ fn verify_upgrade( i += 1; while node.index != iter.index() { if iter.factor() == 2 { - return Err(anyhow!("Unexpected node: {}", node.index)); + return Err(HypercoreError::InvalidOperation { + context: format!("Unexpected node: {}, store: {}", node.index, Store::Tree), + }); } iter.left_child(); } @@ -1486,17 +1521,17 @@ struct LocalProof { pub additional_upgrade: Option>, } -fn nodes_to_root(index: u64, nodes: u64, head: u64) -> Result { +fn nodes_to_root(index: u64, nodes: u64, head: u64) -> Result { let mut iter = flat_tree::Iterator::new(index); for _ in 0..nodes { iter.parent(); if iter.contains(head) { - return Err(anyhow!( - "Nodes is out of bounds, index: {}, nodes: {}, head {}", - index, - nodes, - head - )); + return Err(HypercoreError::InvalidOperation { + context: format!( + "Nodes is out of bounds, index: {}, nodes: {}, head {}", + index, nodes, head + ), + }); } } Ok(iter.index()) @@ -1535,7 +1570,7 @@ impl NodeQueue { length, } } - pub fn shift(&mut self, index: u64) -> Result { + pub fn shift(&mut self, index: u64) -> Result { if let Some(extra) = self.extra.take() { if extra.index == index { self.length -= 1; @@ -1545,12 +1580,16 @@ impl NodeQueue { } } if self.i >= self.nodes.len() { - return Err(anyhow!("Expected node {}, got (nil)", index)); + return Err(HypercoreError::InvalidOperation { + context: format!("Expected node {}, got (nil)", index), + }); } let node = self.nodes[self.i].clone(); self.i += 1; if node.index != index { - return Err(anyhow!("Expected node {}, got node {}", index, node.index)); + return Err(HypercoreError::InvalidOperation { + context: format!("Expected node {}, got node {}", index, node.index), + }); } self.length -= 1; Ok(node) diff --git a/src/tree/merkle_tree_changeset.rs b/src/tree/merkle_tree_changeset.rs index b72440e3..291f104a 100644 --- a/src/tree/merkle_tree_changeset.rs +++ b/src/tree/merkle_tree_changeset.rs @@ -3,7 +3,7 @@ use std::convert::TryFrom; use crate::{ crypto::{signable_tree, verify, Hash}, - sign, Node, + sign, HypercoreError, Node, }; /// Changeset for a `MerkleTree`. This allows to incrementally change a `MerkleTree` in two steps: @@ -99,9 +99,12 @@ impl MerkleTreeChangeset { &mut self, signature: &[u8], public_key: &PublicKey, - ) -> anyhow::Result<()> { + ) -> Result<(), HypercoreError> { // Verify that the received signature matches the public key - let signature = Signature::try_from(signature)?; + let signature = + Signature::try_from(signature).map_err(|_| HypercoreError::InvalidSignature { + context: "Could not parse signature".to_string(), + })?; let hash = self.hash(); verify(&public_key, &self.signable(&hash), Some(&signature))?; From fb87a491974d336c7ae5cc1344c7f9b6685df989 Mon Sep 17 00:00:00 2001 From: Timo Tiuraniemi Date: Thu, 23 Mar 2023 14:54:09 +0200 Subject: [PATCH 105/157] 4/n switch from anyhow to thiserror --- src/storage/mod.rs | 111 ++++++++++++++++++++++++++++++++------------- 1 file changed, 80 insertions(+), 31 deletions(-) diff --git a/src/storage/mod.rs b/src/storage/mod.rs index dadc1656..aa59bcb5 100644 --- a/src/storage/mod.rs +++ b/src/storage/mod.rs @@ -1,6 +1,5 @@ //! Save data to a desired storage backend. -use anyhow::{anyhow, Result}; use ed25519_dalek::{PublicKey, SecretKey}; use futures::future::FutureExt; #[cfg(not(target_arch = "wasm32"))] @@ -10,7 +9,10 @@ use random_access_storage::{RandomAccess, RandomAccessError}; use std::fmt::Debug; use std::path::PathBuf; -use crate::common::{Store, StoreInfo, StoreInfoInstruction, StoreInfoType}; +use crate::{ + common::{Store, StoreInfo, StoreInfoInstruction, StoreInfoType}, + HypercoreError, +}; /// Key pair where for read-only hypercores the secret key can also be missing. #[derive(Debug)] @@ -49,12 +51,38 @@ where oplog: T, } +pub fn map_random_access_err(err: RandomAccessError) -> HypercoreError { + match err { + RandomAccessError::IO { + return_code, + context, + source, + } => HypercoreError::IO { + context: Some(format!( + "RandomAccess IO error. Context: {:?}, return_code: {:?}", + context, return_code + )), + source, + }, + RandomAccessError::OutOfBounds { + offset, + end, + length, + } => HypercoreError::InvalidOperation { + context: format!( + "RandomAccess out of bounds. Offset: {}, end: {:?}, length: {}", + offset, end, length + ), + }, + } +} + impl Storage where T: RandomAccess + Debug + Send, { /// Create a new instance. Takes a callback to create new storage instances and overwrite flag. - pub async fn open(create: Cb, overwrite: bool) -> Result + pub async fn open(create: Cb, overwrite: bool) -> Result where Cb: Fn( Store, @@ -62,23 +90,25 @@ where Box> + Send>, >, { - let mut tree = create(Store::Tree).await?; - let mut data = create(Store::Data).await?; - let mut bitfield = create(Store::Bitfield).await?; - let mut oplog = create(Store::Oplog).await?; + let mut tree = create(Store::Tree).await.map_err(map_random_access_err)?; + let mut data = create(Store::Data).await.map_err(map_random_access_err)?; + let mut bitfield = create(Store::Bitfield) + .await + .map_err(map_random_access_err)?; + let mut oplog = create(Store::Oplog).await.map_err(map_random_access_err)?; if overwrite { - if tree.len().await.map_err(|e| anyhow!(e))? > 0 { - tree.truncate(0).await.map_err(|e| anyhow!(e))?; + if tree.len().await.map_err(map_random_access_err)? > 0 { + tree.truncate(0).await.map_err(map_random_access_err)?; } - if data.len().await.map_err(|e| anyhow!(e))? > 0 { - data.truncate(0).await.map_err(|e| anyhow!(e))?; + if data.len().await.map_err(map_random_access_err)? > 0 { + data.truncate(0).await.map_err(map_random_access_err)?; } - if bitfield.len().await.map_err(|e| anyhow!(e))? > 0 { - bitfield.truncate(0).await.map_err(|e| anyhow!(e))?; + if bitfield.len().await.map_err(map_random_access_err)? > 0 { + bitfield.truncate(0).await.map_err(map_random_access_err)?; } - if oplog.len().await.map_err(|e| anyhow!(e))? > 0 { - oplog.truncate(0).await.map_err(|e| anyhow!(e))?; + if oplog.len().await.map_err(map_random_access_err)? > 0 { + oplog.truncate(0).await.map_err(map_random_access_err)?; } } @@ -93,7 +123,10 @@ where } /// Read info from store based on given instruction. Convenience method to `read_infos`. - pub async fn read_info(&mut self, info_instruction: StoreInfoInstruction) -> Result { + pub async fn read_info( + &mut self, + info_instruction: StoreInfoInstruction, + ) -> Result { let mut infos = self.read_infos_to_vec(&[info_instruction]).await?; Ok(infos .pop() @@ -104,7 +137,7 @@ where pub async fn read_infos( &mut self, info_instructions: &[StoreInfoInstruction], - ) -> Result> { + ) -> Result, HypercoreError> { let infos = self.read_infos_to_vec(info_instructions).await?; Ok(infos.into_boxed_slice()) } @@ -113,7 +146,7 @@ where pub async fn read_infos_to_vec( &mut self, info_instructions: &[StoreInfoInstruction], - ) -> Result> { + ) -> Result, HypercoreError> { if info_instructions.is_empty() { return Ok(vec![]); } @@ -127,32 +160,45 @@ where } match instruction.info_type { StoreInfoType::Content => { - let length = match instruction.length { + let read_length = match instruction.length { Some(length) => length, - None => storage.len().await.map_err(|e| anyhow!(e))?, + None => storage.len().await.map_err(map_random_access_err)?, }; - let read_result = storage.read(instruction.index, length).await; + let read_result = storage.read(instruction.index, read_length).await; let info: StoreInfo = match read_result { Ok(buf) => Ok(StoreInfo::new_content( instruction.store.clone(), instruction.index, &buf, )), - Err(e) => { + Err(RandomAccessError::OutOfBounds { + offset: _, + end: _, + length, + }) => { if instruction.allow_miss { Ok(StoreInfo::new_content_miss( instruction.store.clone(), instruction.index, )) } else { - Err(anyhow!(e)) + Err(HypercoreError::InvalidOperation { + context: format!( + "Could not read from store {}, index {} / length {} is out of bounds for store length {}", + instruction.index, + read_length, + current_store, + length + ), + }) } } + Err(e) => Err(map_random_access_err(e)), }?; infos.push(info); } StoreInfoType::Size => { - let length = storage.len().await.map_err(|e| anyhow!(e))?; + let length = storage.len().await.map_err(map_random_access_err)?; infos.push(StoreInfo::new_size( instruction.store.clone(), instruction.index, @@ -165,12 +211,12 @@ where } /// Flush info to storage. Convenience method to `flush_infos`. - pub async fn flush_info(&mut self, slice: StoreInfo) -> Result<()> { + pub async fn flush_info(&mut self, slice: StoreInfo) -> Result<(), HypercoreError> { self.flush_infos(&[slice]).await } /// Flush infos to storage - pub async fn flush_infos(&mut self, infos: &[StoreInfo]) -> Result<()> { + pub async fn flush_infos(&mut self, infos: &[StoreInfo]) -> Result<(), HypercoreError> { if infos.is_empty() { return Ok(()); } @@ -188,7 +234,7 @@ where storage .write(info.index, &data.to_vec()) .await - .map_err(|e| anyhow!(e))?; + .map_err(map_random_access_err)?; } } else { storage @@ -197,12 +243,15 @@ where info.length.expect("When deleting, length must be given"), ) .await - .map_err(|e| anyhow!(e))?; + .map_err(map_random_access_err)?; } } StoreInfoType::Size => { if info.miss { - storage.truncate(info.index).await.map_err(|e| anyhow!(e))?; + storage + .truncate(info.index) + .await + .map_err(map_random_access_err)?; } else { panic!("Flushing a size that isn't miss, is not supported"); } @@ -224,7 +273,7 @@ where impl Storage { /// New storage backed by a `RandomAccessMemory` instance. - pub async fn new_memory() -> Result { + pub async fn new_memory() -> Result { let create = |_| async { Ok(RandomAccessMemory::default()) }.boxed(); // No reason to overwrite, as this is a new memory segment Ok(Self::open(create, false).await?) @@ -234,7 +283,7 @@ impl Storage { #[cfg(not(target_arch = "wasm32"))] impl Storage { /// New storage backed by a `RandomAccessDisk` instance. - pub async fn new_disk(dir: &PathBuf, overwrite: bool) -> Result { + pub async fn new_disk(dir: &PathBuf, overwrite: bool) -> Result { let storage = |store: Store| { let name = match store { Store::Tree => "tree", From 5724f6cda61030eef6d78b382fc9e22e689e6cbd Mon Sep 17 00:00:00 2001 From: Timo Tiuraniemi Date: Thu, 23 Mar 2023 15:15:15 +0200 Subject: [PATCH 106/157] 5/5 switch from anyhow to thiserror --- Cargo.toml | 2 +- src/common/error.rs | 3 ++ src/core.rs | 127 +++++++++++++++++++++++++++++--------------- 3 files changed, 87 insertions(+), 45 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 6fcee19e..3fe9d21d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -26,7 +26,6 @@ blake2-rfc = "0.2.18" byteorder = "1.3.4" ed25519-dalek = "1.0.1" thiserror = "1" -anyhow = "1.0.26" compact-encoding = { git = "https://github.com/datrs/compact-encoding", branch = "main" } flat-tree = { git = "https://github.com/ttiurani/flat-tree", branch = "v10" } lazy_static = "1.4.0" @@ -50,6 +49,7 @@ intmap = "2.0.0" random-access-disk = { git = "https://github.com/ttiurani/random-access-disk", branch = "thiserror" } [dev-dependencies] +anyhow = "1.0.70" quickcheck = "0.9.2" data-encoding = "2.2.0" remove_dir_all = "0.7.0" diff --git a/src/common/error.rs b/src/common/error.rs index fc11458d..01888090 100644 --- a/src/common/error.rs +++ b/src/common/error.rs @@ -11,6 +11,9 @@ pub enum HypercoreError { /// Context for the error context: String, }, + /// Not writable + #[error("Hypercore not writable")] + NotWritable, /// Invalid signature #[error("Given signature was invalid. {context}")] InvalidSignature { diff --git a/src/core.rs b/src/core.rs index 097f8761..c8dcfad1 100644 --- a/src/core.rs +++ b/src/core.rs @@ -2,7 +2,7 @@ use crate::{ bitfield::Bitfield, - common::{BitfieldUpdate, Proof, StoreInfo, ValuelessProof}, + common::{BitfieldUpdate, HypercoreError, Proof, StoreInfo, ValuelessProof}, crypto::generate_keypair, data::BlockStore, oplog::{Header, Oplog, MAX_OPLOG_ENTRIES_BYTE_SIZE}, @@ -10,7 +10,6 @@ use crate::{ tree::{MerkleTree, MerkleTreeChangeset}, RequestBlock, RequestSeek, RequestUpgrade, }; -use anyhow::{anyhow, Result}; use ed25519_dalek::Signature; use futures::future::Either; use random_access_storage::RandomAccess; @@ -55,7 +54,7 @@ where T: RandomAccess + Debug + Send, { /// Creates new hypercore using given storage with random key pair - pub async fn new(storage: Storage) -> Result> { + pub async fn new(storage: Storage) -> Result, HypercoreError> { let key_pair = generate_keypair(); Hypercore::new_with_key_pair( storage, @@ -71,12 +70,12 @@ where pub async fn new_with_key_pair( storage: Storage, key_pair: PartialKeypair, - ) -> Result> { + ) -> Result, HypercoreError> { Hypercore::resume(storage, Some(key_pair)).await } /// Opens an existing hypercore in given storage. - pub async fn open(storage: Storage) -> Result> { + pub async fn open(storage: Storage) -> Result, HypercoreError> { Hypercore::resume(storage, None).await } @@ -84,7 +83,7 @@ where async fn resume( mut storage: Storage, key_pair: Option, - ) -> Result> { + ) -> Result, HypercoreError> { // Open/create oplog let mut oplog_open_outcome = match Oplog::open(&key_pair, None)? { Either::Right(value) => value, @@ -93,7 +92,9 @@ where match Oplog::open(&key_pair, Some(info))? { Either::Right(value) => value, Either::Left(_) => { - return Err(anyhow!("Could not open tree")); + return Err(HypercoreError::InvalidOperation { + context: "Could not open oplog".to_string(), + }); } } } @@ -110,7 +111,9 @@ where match MerkleTree::open(&oplog_open_outcome.header.tree, Some(&infos))? { Either::Right(value) => value, Either::Left(_) => { - return Err(anyhow!("Could not open tree")); + return Err(HypercoreError::InvalidOperation { + context: "Could not open tree".to_string(), + }); } } } @@ -131,7 +134,9 @@ where match Bitfield::open(Some(info)) { Either::Right(value) => value, Either::Left(_) => { - return Err(anyhow!("Could not open bitfield")); + return Err(HypercoreError::InvalidOperation { + context: "Could not open bitfield".to_string(), + }); } } } @@ -168,13 +173,23 @@ where )? { Either::Right(value) => value, Either::Left(_) => { - return Err(anyhow!("Could not truncate")); + return Err(HypercoreError::InvalidOperation { + context: format!( + "Could not truncate tree to length {}", + tree_upgrade.length + ), + }); } } } }; changeset.ancestors = tree_upgrade.ancestors; - changeset.signature = Some(Signature::try_from(&*tree_upgrade.signature)?); + changeset.signature = + Some(Signature::try_from(&*tree_upgrade.signature).map_err(|_| { + HypercoreError::InvalidSignature { + context: "Could not parse changeset signature".to_string(), + } + })?); // TODO: Skip reorg hints for now, seems to only have to do with replication // addReorgHint(header.hints.reorgs, tree, batch) @@ -213,15 +228,15 @@ where } /// Appends a data slice to the hypercore. - pub async fn append(&mut self, data: &[u8]) -> Result { + pub async fn append(&mut self, data: &[u8]) -> Result { self.append_batch(&[data]).await } /// Appends a given batch of data slices to the hypercore. - pub async fn append_batch(&mut self, batch: &[&[u8]]) -> Result { + pub async fn append_batch(&mut self, batch: &[&[u8]]) -> Result { let secret_key = match &self.key_pair.secret { Some(key) => key, - None => anyhow::bail!("No secret key, cannot append."), + None => return Err(HypercoreError::NotWritable), }; if !batch.is_empty() { @@ -277,7 +292,7 @@ where } /// Read value at given index, if any. - pub async fn get(&mut self, index: u64) -> Result>> { + pub async fn get(&mut self, index: u64) -> Result>, HypercoreError> { if !self.bitfield.get(index) { return Ok(None); } @@ -290,7 +305,12 @@ where match self.tree.byte_range(index, Some(&infos))? { Either::Right(value) => value, Either::Left(_) => { - return Err(anyhow!("Could not read byte range")); + return Err(HypercoreError::InvalidOperation { + context: format!( + "Could not read byte range at index {} from tree", + index + ), + }); } } } @@ -304,7 +324,9 @@ where match self.block_store.read(&byte_range, Some(info)) { Either::Right(value) => value, Either::Left(_) => { - return Err(anyhow!("Could not read block storage range")); + return Err(HypercoreError::InvalidOperation { + context: "Could not read block storage range".to_string(), + }); } } } @@ -314,7 +336,7 @@ where } /// Clear data for entries between start and end (exclusive) indexes. - pub async fn clear(&mut self, start: u64, end: u64) -> Result<()> { + pub async fn clear(&mut self, start: u64, end: u64) -> Result<(), HypercoreError> { if start >= end { // NB: This is what javascript does, so we mimic that here return Ok(()); @@ -353,7 +375,9 @@ where match self.tree.byte_offset(start, Some(&infos))? { Either::Right(value) => value, Either::Left(_) => { - return Err(anyhow!("Could not read offset for index")); + return Err(HypercoreError::InvalidOperation { + context: format!("Could not read offset for index {} from tree", start), + }); } } } @@ -368,7 +392,12 @@ where match self.tree.byte_range(end - 1, Some(&infos))? { Either::Right(value) => value, Either::Left(_) => { - return Err(anyhow!("Could not read byte range")); + return Err(HypercoreError::InvalidOperation { + context: format!( + "Could not read byte range for index {} from tree", + end - 1 + ), + }); } } } @@ -399,7 +428,7 @@ where hash: Option, seek: Option, upgrade: Option, - ) -> Result> { + ) -> Result, HypercoreError> { let mut valueless_proof = self .create_valueless_proof(block, hash, seek, upgrade) .await?; @@ -419,7 +448,7 @@ where /// Verify and apply proof received from peer, returns true if changed, false if not /// possible to apply. - pub async fn verify_and_apply_proof(&mut self, proof: &Proof) -> Result { + pub async fn verify_and_apply_proof(&mut self, proof: &Proof) -> Result { if proof.fork != self.tree.fork { return Ok(false); } @@ -447,7 +476,12 @@ where )? { Either::Right(value) => value, Either::Left(_) => { - return Err(anyhow!("Could not read offset for index")); + return Err(HypercoreError::InvalidOperation { + context: format!( + "Could not read offset for index {} from tree", + block.index + ), + }); } } } @@ -495,7 +529,7 @@ where /// Used to fill the nodes field of a `RequestBlock` during /// synchronization. - pub async fn missing_nodes(&mut self, merkle_tree_index: u64) -> Result { + pub async fn missing_nodes(&mut self, merkle_tree_index: u64) -> Result { match self.tree.missing_nodes(merkle_tree_index, None)? { Either::Right(value) => return Ok(value), Either::Left(instructions) => { @@ -522,7 +556,7 @@ where hash: Option, seek: Option, upgrade: Option, - ) -> Result { + ) -> Result { match self.tree.create_valueless_proof( block.as_ref(), hash.as_ref(), @@ -557,7 +591,7 @@ where /// Verify a proof received from a peer. Returns a changeset that should be /// applied. - async fn verify_proof(&mut self, proof: &Proof) -> Result { + async fn verify_proof(&mut self, proof: &Proof) -> Result { match self.tree.verify_proof(proof, &self.key_pair.public, None)? { Either::Right(value) => Ok(value), Either::Left(instructions) => { @@ -568,7 +602,9 @@ where { Either::Right(value) => Ok(value), Either::Left(_) => { - return Err(anyhow!("Could not verify key pair")); + return Err(HypercoreError::InvalidOperation { + context: format!("Could not verify proof from tree"), + }); } } } @@ -587,7 +623,7 @@ where } } - async fn flush_bitfield_and_tree_and_oplog(&mut self) -> Result<()> { + async fn flush_bitfield_and_tree_and_oplog(&mut self) -> Result<(), HypercoreError> { let infos = self.bitfield.flush(); self.storage.flush_infos(&infos).await?; let infos = self.tree.flush(); @@ -629,7 +665,7 @@ mod tests { use random_access_memory::RandomAccessMemory; #[async_std::test] - async fn core_create_proof_block_only() -> Result<()> { + async fn core_create_proof_block_only() -> Result<(), HypercoreError> { let mut hypercore = create_hypercore_with_data(10).await?; let proof = hypercore @@ -647,7 +683,7 @@ mod tests { } #[async_std::test] - async fn core_create_proof_block_and_upgrade() -> Result<()> { + async fn core_create_proof_block_and_upgrade() -> Result<(), HypercoreError> { let mut hypercore = create_hypercore_with_data(10).await?; let proof = hypercore .create_proof( @@ -678,7 +714,7 @@ mod tests { } #[async_std::test] - async fn core_create_proof_block_and_upgrade_and_additional() -> Result<()> { + async fn core_create_proof_block_and_upgrade_and_additional() -> Result<(), HypercoreError> { let mut hypercore = create_hypercore_with_data(10).await?; let proof = hypercore .create_proof( @@ -709,7 +745,8 @@ mod tests { } #[async_std::test] - async fn core_create_proof_block_and_upgrade_from_existing_state() -> Result<()> { + async fn core_create_proof_block_and_upgrade_from_existing_state() -> Result<(), HypercoreError> + { let mut hypercore = create_hypercore_with_data(10).await?; let proof = hypercore .create_proof( @@ -739,8 +776,8 @@ mod tests { } #[async_std::test] - async fn core_create_proof_block_and_upgrade_from_existing_state_with_additional() -> Result<()> - { + async fn core_create_proof_block_and_upgrade_from_existing_state_with_additional( + ) -> Result<(), HypercoreError> { let mut hypercore = create_hypercore_with_data(10).await?; let proof = hypercore .create_proof( @@ -771,7 +808,7 @@ mod tests { } #[async_std::test] - async fn core_create_proof_block_and_seek_1_no_upgrade() -> Result<()> { + async fn core_create_proof_block_and_seek_1_no_upgrade() -> Result<(), HypercoreError> { let mut hypercore = create_hypercore_with_data(10).await?; let proof = hypercore .create_proof( @@ -793,7 +830,7 @@ mod tests { } #[async_std::test] - async fn core_create_proof_block_and_seek_2_no_upgrade() -> Result<()> { + async fn core_create_proof_block_and_seek_2_no_upgrade() -> Result<(), HypercoreError> { let mut hypercore = create_hypercore_with_data(10).await?; let proof = hypercore .create_proof( @@ -815,7 +852,7 @@ mod tests { } #[async_std::test] - async fn core_create_proof_block_and_seek_3_no_upgrade() -> Result<()> { + async fn core_create_proof_block_and_seek_3_no_upgrade() -> Result<(), HypercoreError> { let mut hypercore = create_hypercore_with_data(10).await?; let proof = hypercore .create_proof( @@ -839,7 +876,7 @@ mod tests { } #[async_std::test] - async fn core_create_proof_block_and_seek_to_tree_no_upgrade() -> Result<()> { + async fn core_create_proof_block_and_seek_to_tree_no_upgrade() -> Result<(), HypercoreError> { let mut hypercore = create_hypercore_with_data(16).await?; let proof = hypercore .create_proof( @@ -864,7 +901,7 @@ mod tests { } #[async_std::test] - async fn core_create_proof_block_and_seek_with_upgrade() -> Result<()> { + async fn core_create_proof_block_and_seek_with_upgrade() -> Result<(), HypercoreError> { let mut hypercore = create_hypercore_with_data(10).await?; let proof = hypercore .create_proof( @@ -894,7 +931,7 @@ mod tests { } #[async_std::test] - async fn core_create_proof_seek_with_upgrade() -> Result<()> { + async fn core_create_proof_seek_with_upgrade() -> Result<(), HypercoreError> { let mut hypercore = create_hypercore_with_data(10).await?; let proof = hypercore .create_proof( @@ -923,7 +960,7 @@ mod tests { } #[async_std::test] - async fn core_verify_proof_invalid_signature() -> Result<()> { + async fn core_verify_proof_invalid_signature() -> Result<(), HypercoreError> { let mut hypercore = create_hypercore_with_data(10).await?; // Invalid clone hypercore with a different public key let mut hypercore_clone = create_hypercore_with_data(0).await?; @@ -947,7 +984,7 @@ mod tests { } #[async_std::test] - async fn core_verify_and_apply_proof() -> Result<()> { + async fn core_verify_and_apply_proof() -> Result<(), HypercoreError> { let mut main = create_hypercore_with_data(10).await?; let mut clone = create_hypercore_with_data_and_key_pair( 0, @@ -990,7 +1027,9 @@ mod tests { Ok(()) } - async fn create_hypercore_with_data(length: u64) -> Result> { + async fn create_hypercore_with_data( + length: u64, + ) -> Result, HypercoreError> { let key_pair = generate_keypair(); Ok(create_hypercore_with_data_and_key_pair( length, @@ -1005,7 +1044,7 @@ mod tests { async fn create_hypercore_with_data_and_key_pair( length: u64, key_pair: PartialKeypair, - ) -> Result> { + ) -> Result, HypercoreError> { let storage = Storage::new_memory().await?; let mut hypercore = Hypercore::new_with_key_pair(storage, key_pair).await?; for i in 0..length { From 0f1d5f4c6e2fdeb3a914b1ae23bb14c2f24c59ab Mon Sep 17 00:00:00 2001 From: Timo Tiuraniemi Date: Fri, 24 Mar 2023 10:30:24 +0200 Subject: [PATCH 107/157] Remove unused async_std hard dependency --- Cargo.toml | 1 - 1 file changed, 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index 3fe9d21d..31343ed8 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -41,7 +41,6 @@ sparse-bitfield = "0.11.0" tree-index = "0.6.0" bitfield-rle = "0.2.0" futures = "0.3.4" -async-std = "1.5.0" crc32fast = "1.3.2" intmap = "2.0.0" From a884b3f3378409483272bbef59d2a6ccfa3f1df4 Mon Sep 17 00:00:00 2001 From: Timo Tiuraniemi Date: Fri, 24 Mar 2023 11:19:00 +0200 Subject: [PATCH 108/157] Switch to a builder interface to be able to add backward compatible options --- benches/memory.rs | 9 ++--- src/builder.rs | 83 +++++++++++++++++++++++++++++++++++++++++++++ src/core.rs | 6 ++-- src/lib.rs | 2 ++ tests/core.rs | 9 +++-- tests/js_interop.rs | 9 +++-- tests/model.rs | 5 +-- 7 files changed, 108 insertions(+), 15 deletions(-) create mode 100644 src/builder.rs diff --git a/benches/memory.rs b/benches/memory.rs index 7d2a89d5..2f6aac7e 100644 --- a/benches/memory.rs +++ b/benches/memory.rs @@ -1,18 +1,19 @@ use std::time::Instant; -use anyhow::Error; use criterion::async_executor::AsyncStdExecutor; use criterion::{black_box, criterion_group, criterion_main, Criterion}; -use hypercore::{Hypercore, Storage}; +use hypercore::{Builder, Hypercore, HypercoreError, Storage}; use random_access_memory::RandomAccessMemory; -async fn create_hypercore(page_size: usize) -> Result, Error> { +async fn create_hypercore( + page_size: usize, +) -> Result, HypercoreError> { let storage = Storage::open( |_| Box::pin(async move { Ok(RandomAccessMemory::new(page_size)) }), false, ) .await?; - Hypercore::new(storage).await + Builder::new(storage).build_new().await } fn create(c: &mut Criterion) { diff --git a/src/builder.rs b/src/builder.rs new file mode 100644 index 00000000..f84e5dfa --- /dev/null +++ b/src/builder.rs @@ -0,0 +1,83 @@ +use std::fmt::Debug; + +use random_access_storage::RandomAccess; + +use crate::{Hypercore, HypercoreError, PartialKeypair, Storage}; + +/// Options for a Hypercore instance. +#[derive(Debug)] +pub struct Options +where + T: RandomAccess + Debug + Send, +{ + /// Existing key pair to use + pub key_pair: Option, + /// Storage + pub storage: Option>, +} + +impl Options +where + T: RandomAccess + Debug + Send, +{ + /// Create with default options. + pub fn new(storage: Storage) -> Self { + Self { + storage: Some(storage), + key_pair: None, + } + } +} + +/// Build a Hypercore instance with options. +#[derive(Debug)] +pub struct Builder(Options) +where + T: RandomAccess + Debug + Send; + +impl Builder +where + T: RandomAccess + Debug + Send, +{ + /// Create a hypercore builder with a given storage + pub fn new(storage: Storage) -> Self { + Self(Options::new(storage)) + } + + /// Set key pair. + pub fn set_key_pair(mut self, key_pair: PartialKeypair) -> Self { + self.0.key_pair = Some(key_pair); + self + } + + /// Build a new Hypercore. + pub async fn build_new(mut self) -> Result, HypercoreError> { + if let Some(storage) = self.0.storage.take() { + if let Some(key_pair) = self.0.key_pair.take() { + Hypercore::new_with_key_pair(storage, key_pair).await + } else { + Hypercore::new(storage).await + } + } else { + return Err(HypercoreError::BadArgument { + context: "Storage must be provided when building a new hypercore".to_string(), + }); + } + } + + /// Build an existing Hypercore. + pub async fn build_existing(mut self) -> Result, HypercoreError> { + if self.0.key_pair.is_some() { + return Err(HypercoreError::BadArgument { + context: "Key pair can not be used when building an existing hypercore".to_string(), + }); + } + if let Some(storage) = self.0.storage.take() { + Hypercore::open(storage).await + } else { + return Err(HypercoreError::BadArgument { + context: "Storage must be provided when building an existing hypercore".to_string(), + }); + } + } +} diff --git a/src/core.rs b/src/core.rs index c8dcfad1..6be7e6db 100644 --- a/src/core.rs +++ b/src/core.rs @@ -54,7 +54,7 @@ where T: RandomAccess + Debug + Send, { /// Creates new hypercore using given storage with random key pair - pub async fn new(storage: Storage) -> Result, HypercoreError> { + pub(crate) async fn new(storage: Storage) -> Result, HypercoreError> { let key_pair = generate_keypair(); Hypercore::new_with_key_pair( storage, @@ -67,7 +67,7 @@ where } /// Creates new hypercore with given storage and (partial) key pair - pub async fn new_with_key_pair( + pub(crate) async fn new_with_key_pair( storage: Storage, key_pair: PartialKeypair, ) -> Result, HypercoreError> { @@ -75,7 +75,7 @@ where } /// Opens an existing hypercore in given storage. - pub async fn open(storage: Storage) -> Result, HypercoreError> { + pub(crate) async fn open(storage: Storage) -> Result, HypercoreError> { Hypercore::resume(storage, None).await } diff --git a/src/lib.rs b/src/lib.rs index f539fe45..5852fa64 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -29,6 +29,7 @@ pub mod encoding; pub mod prelude; mod bitfield; +mod builder; mod common; mod core; mod crypto; @@ -37,6 +38,7 @@ mod oplog; mod storage; mod tree; +pub use crate::builder::Builder; pub use crate::common::{ DataBlock, DataHash, DataSeek, DataUpgrade, HypercoreError, Node, Proof, RequestBlock, RequestSeek, RequestUpgrade, Store, diff --git a/tests/core.rs b/tests/core.rs index 41085471..de2713a6 100644 --- a/tests/core.rs +++ b/tests/core.rs @@ -2,12 +2,12 @@ pub mod common; use anyhow::Result; use common::get_test_key_pair; -use hypercore::{Hypercore, Storage}; +use hypercore::{Builder, Storage}; #[async_std::test] async fn hypercore_new() -> Result<()> { let storage = Storage::new_memory().await?; - let _hypercore = Hypercore::new(storage).await?; + let _hypercore = Builder::new(storage).build_new(); Ok(()) } @@ -15,6 +15,9 @@ async fn hypercore_new() -> Result<()> { async fn hypercore_new_with_key_pair() -> Result<()> { let storage = Storage::new_memory().await?; let key_pair = get_test_key_pair(); - let _hypercore = Hypercore::new_with_key_pair(storage, key_pair).await?; + let _hypercore = Builder::new(storage) + .set_key_pair(key_pair) + .build_new() + .await?; Ok(()) } diff --git a/tests/js_interop.rs b/tests/js_interop.rs index da7589e0..72eafb39 100644 --- a/tests/js_interop.rs +++ b/tests/js_interop.rs @@ -4,7 +4,7 @@ use std::{path::Path, sync::Once}; use anyhow::Result; use common::{create_hypercore_hash, get_test_key_pair}; -use hypercore::{Hypercore, Storage}; +use hypercore::{Builder, Hypercore, Storage}; use js::{cleanup, install, js_run_step, prepare_test_set}; use random_access_disk::RandomAccessDisk; @@ -139,13 +139,16 @@ async fn create_hypercore(work_dir: &str) -> Result> let path = Path::new(work_dir).to_owned(); let key_pair = get_test_key_pair(); let storage = Storage::new_disk(&path, true).await?; - Ok(Hypercore::new_with_key_pair(storage, key_pair).await?) + Ok(Builder::new(storage) + .set_key_pair(key_pair) + .build_new() + .await?) } async fn open_hypercore(work_dir: &str) -> Result> { let path = Path::new(work_dir).to_owned(); let storage = Storage::new_disk(&path, false).await?; - Ok(Hypercore::open(storage).await?) + Ok(Builder::new(storage).build_existing().await?) } fn step_0_hash() -> common::HypercoreHash { diff --git a/tests/model.rs b/tests/model.rs index 5e80f847..23ed37eb 100644 --- a/tests/model.rs +++ b/tests/model.rs @@ -66,12 +66,13 @@ quickcheck! { } async fn assert_implementation_matches_model(ops: Vec) -> bool { - use hypercore::{Hypercore, Storage}; + use hypercore::{Builder, Storage}; let storage = Storage::new_memory() .await .expect("Memory storage creation should be successful"); - let mut hypercore = Hypercore::new(storage) + let mut hypercore = Builder::new(storage) + .build_new() .await .expect("Hypercore creation should be successful"); From 34a0f3a292d8b2cdc2b59ab38a623e78124a5bcc Mon Sep 17 00:00:00 2001 From: Timo Tiuraniemi Date: Fri, 24 Mar 2023 11:30:29 +0200 Subject: [PATCH 109/157] Simplify builder with just one build() method --- benches/memory.rs | 2 +- src/builder.rs | 48 ++++++++++++++++++++++++--------------------- tests/core.rs | 7 ++----- tests/js_interop.rs | 7 ++----- tests/model.rs | 2 +- 5 files changed, 32 insertions(+), 34 deletions(-) diff --git a/benches/memory.rs b/benches/memory.rs index 2f6aac7e..cffcbe19 100644 --- a/benches/memory.rs +++ b/benches/memory.rs @@ -13,7 +13,7 @@ async fn create_hypercore( false, ) .await?; - Builder::new(storage).build_new().await + Builder::new(storage).build().await } fn create(c: &mut Criterion) { diff --git a/src/builder.rs b/src/builder.rs index f84e5dfa..95218c9e 100644 --- a/src/builder.rs +++ b/src/builder.rs @@ -14,6 +14,8 @@ where pub key_pair: Option, /// Storage pub storage: Option>, + /// Whether or not to open existing or create new + pub open: bool, } impl Options @@ -25,6 +27,7 @@ where Self { storage: Some(storage), key_pair: None, + open: false, } } } @@ -50,34 +53,35 @@ where self } + /// Set open. + pub fn set_open(mut self, open: bool) -> Self { + self.0.open = open; + self + } + /// Build a new Hypercore. - pub async fn build_new(mut self) -> Result, HypercoreError> { - if let Some(storage) = self.0.storage.take() { + pub async fn build(mut self) -> Result, HypercoreError> { + let storage = self + .0 + .storage + .take() + .ok_or_else(|| HypercoreError::BadArgument { + context: "Storage must be provided when building a hypercore".to_string(), + })?; + if self.0.open { + if self.0.key_pair.is_some() { + return Err(HypercoreError::BadArgument { + context: "Key pair can not be used when building an openable hypercore" + .to_string(), + }); + } + Hypercore::open(storage).await + } else { if let Some(key_pair) = self.0.key_pair.take() { Hypercore::new_with_key_pair(storage, key_pair).await } else { Hypercore::new(storage).await } - } else { - return Err(HypercoreError::BadArgument { - context: "Storage must be provided when building a new hypercore".to_string(), - }); - } - } - - /// Build an existing Hypercore. - pub async fn build_existing(mut self) -> Result, HypercoreError> { - if self.0.key_pair.is_some() { - return Err(HypercoreError::BadArgument { - context: "Key pair can not be used when building an existing hypercore".to_string(), - }); - } - if let Some(storage) = self.0.storage.take() { - Hypercore::open(storage).await - } else { - return Err(HypercoreError::BadArgument { - context: "Storage must be provided when building an existing hypercore".to_string(), - }); } } } diff --git a/tests/core.rs b/tests/core.rs index de2713a6..1759620c 100644 --- a/tests/core.rs +++ b/tests/core.rs @@ -7,7 +7,7 @@ use hypercore::{Builder, Storage}; #[async_std::test] async fn hypercore_new() -> Result<()> { let storage = Storage::new_memory().await?; - let _hypercore = Builder::new(storage).build_new(); + let _hypercore = Builder::new(storage).build(); Ok(()) } @@ -15,9 +15,6 @@ async fn hypercore_new() -> Result<()> { async fn hypercore_new_with_key_pair() -> Result<()> { let storage = Storage::new_memory().await?; let key_pair = get_test_key_pair(); - let _hypercore = Builder::new(storage) - .set_key_pair(key_pair) - .build_new() - .await?; + let _hypercore = Builder::new(storage).set_key_pair(key_pair).build().await?; Ok(()) } diff --git a/tests/js_interop.rs b/tests/js_interop.rs index 72eafb39..e65a9530 100644 --- a/tests/js_interop.rs +++ b/tests/js_interop.rs @@ -139,16 +139,13 @@ async fn create_hypercore(work_dir: &str) -> Result> let path = Path::new(work_dir).to_owned(); let key_pair = get_test_key_pair(); let storage = Storage::new_disk(&path, true).await?; - Ok(Builder::new(storage) - .set_key_pair(key_pair) - .build_new() - .await?) + Ok(Builder::new(storage).set_key_pair(key_pair).build().await?) } async fn open_hypercore(work_dir: &str) -> Result> { let path = Path::new(work_dir).to_owned(); let storage = Storage::new_disk(&path, false).await?; - Ok(Builder::new(storage).build_existing().await?) + Ok(Builder::new(storage).set_open(true).build().await?) } fn step_0_hash() -> common::HypercoreHash { diff --git a/tests/model.rs b/tests/model.rs index 23ed37eb..fe6f4e68 100644 --- a/tests/model.rs +++ b/tests/model.rs @@ -72,7 +72,7 @@ async fn assert_implementation_matches_model(ops: Vec) -> bool { .await .expect("Memory storage creation should be successful"); let mut hypercore = Builder::new(storage) - .build_new() + .build() .await .expect("Hypercore creation should be successful"); From 6a756392179890d01bd03ca902cd0414841f2a5a Mon Sep 17 00:00:00 2001 From: Timo Tiuraniemi Date: Fri, 24 Mar 2023 12:42:08 +0200 Subject: [PATCH 110/157] Add basic tracing error instrumentation to public interface methods --- Cargo.toml | 3 +++ src/builder.rs | 5 +++-- src/core.rs | 8 ++++++++ src/storage/mod.rs | 15 +++++++++------ tests/core.rs | 18 ++++++++++++++++-- tests/js_interop.rs | 5 +++-- 6 files changed, 42 insertions(+), 12 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 31343ed8..98f905f6 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -26,6 +26,7 @@ blake2-rfc = "0.2.18" byteorder = "1.3.4" ed25519-dalek = "1.0.1" thiserror = "1" +tracing = "0.1" compact-encoding = { git = "https://github.com/datrs/compact-encoding", branch = "main" } flat-tree = { git = "https://github.com/ttiurani/flat-tree", branch = "v10" } lazy_static = "1.4.0" @@ -57,6 +58,8 @@ async-std = { version = "1.12.0", features = ["attributes"] } tokio = { version = "1.25.0", default-features = false, features = ["macros", "rt", "rt-multi-thread"] } sha2 = "0.10.2" criterion = { version = "0.4", features = ["async_std"] } +test-log = { version = "0.2.11", default-features = false, features = ["trace"] } +tracing-subscriber = { version = "0.3.16", features = ["env-filter", "fmt"] } [features] default = ["async-std"] diff --git a/src/builder.rs b/src/builder.rs index 95218c9e..e59e9708 100644 --- a/src/builder.rs +++ b/src/builder.rs @@ -1,6 +1,6 @@ -use std::fmt::Debug; - use random_access_storage::RandomAccess; +use std::fmt::Debug; +use tracing::{debug, instrument}; use crate::{Hypercore, HypercoreError, PartialKeypair, Storage}; @@ -60,6 +60,7 @@ where } /// Build a new Hypercore. + #[instrument(err, skip_all)] pub async fn build(mut self) -> Result, HypercoreError> { let storage = self .0 diff --git a/src/core.rs b/src/core.rs index 6be7e6db..9784ffba 100644 --- a/src/core.rs +++ b/src/core.rs @@ -15,6 +15,7 @@ use futures::future::Either; use random_access_storage::RandomAccess; use std::convert::TryFrom; use std::fmt::Debug; +use tracing::instrument; /// Hypercore is an append-only log structure. #[derive(Debug)] @@ -228,11 +229,13 @@ where } /// Appends a data slice to the hypercore. + #[instrument(err, skip_all, fields(data_len = data.len()))] pub async fn append(&mut self, data: &[u8]) -> Result { self.append_batch(&[data]).await } /// Appends a given batch of data slices to the hypercore. + #[instrument(err, skip_all, fields(batch_len = batch.len()))] pub async fn append_batch(&mut self, batch: &[&[u8]]) -> Result { let secret_key = match &self.key_pair.secret { Some(key) => key, @@ -292,6 +295,7 @@ where } /// Read value at given index, if any. + #[instrument(err, skip(self))] pub async fn get(&mut self, index: u64) -> Result>, HypercoreError> { if !self.bitfield.get(index) { return Ok(None); @@ -336,6 +340,7 @@ where } /// Clear data for entries between start and end (exclusive) indexes. + #[instrument(err, skip(self))] pub async fn clear(&mut self, start: u64, end: u64) -> Result<(), HypercoreError> { if start >= end { // NB: This is what javascript does, so we mimic that here @@ -422,6 +427,7 @@ where } /// Create a proof for given request + #[instrument(err, skip_all)] pub async fn create_proof( &mut self, block: Option, @@ -448,6 +454,7 @@ where /// Verify and apply proof received from peer, returns true if changed, false if not /// possible to apply. + #[instrument(skip_all)] pub async fn verify_and_apply_proof(&mut self, proof: &Proof) -> Result { if proof.fork != self.tree.fork { return Ok(false); @@ -529,6 +536,7 @@ where /// Used to fill the nodes field of a `RequestBlock` during /// synchronization. + #[instrument(err, skip(self))] pub async fn missing_nodes(&mut self, merkle_tree_index: u64) -> Result { match self.tree.missing_nodes(merkle_tree_index, None)? { Either::Right(value) => return Ok(value), diff --git a/src/storage/mod.rs b/src/storage/mod.rs index aa59bcb5..67727731 100644 --- a/src/storage/mod.rs +++ b/src/storage/mod.rs @@ -8,6 +8,7 @@ use random_access_memory::RandomAccessMemory; use random_access_storage::{RandomAccess, RandomAccessError}; use std::fmt::Debug; use std::path::PathBuf; +use tracing::instrument; use crate::{ common::{Store, StoreInfo, StoreInfoInstruction, StoreInfoType}, @@ -82,7 +83,7 @@ where T: RandomAccess + Debug + Send, { /// Create a new instance. Takes a callback to create new storage instances and overwrite flag. - pub async fn open(create: Cb, overwrite: bool) -> Result + pub(crate) async fn open(create: Cb, overwrite: bool) -> Result where Cb: Fn( Store, @@ -123,7 +124,7 @@ where } /// Read info from store based on given instruction. Convenience method to `read_infos`. - pub async fn read_info( + pub(crate) async fn read_info( &mut self, info_instruction: StoreInfoInstruction, ) -> Result { @@ -134,7 +135,7 @@ where } /// Read infos from stores based on given instructions - pub async fn read_infos( + pub(crate) async fn read_infos( &mut self, info_instructions: &[StoreInfoInstruction], ) -> Result, HypercoreError> { @@ -143,7 +144,7 @@ where } /// Reads infos but retains them as a Vec - pub async fn read_infos_to_vec( + pub(crate) async fn read_infos_to_vec( &mut self, info_instructions: &[StoreInfoInstruction], ) -> Result, HypercoreError> { @@ -211,12 +212,12 @@ where } /// Flush info to storage. Convenience method to `flush_infos`. - pub async fn flush_info(&mut self, slice: StoreInfo) -> Result<(), HypercoreError> { + pub(crate) async fn flush_info(&mut self, slice: StoreInfo) -> Result<(), HypercoreError> { self.flush_infos(&[slice]).await } /// Flush infos to storage - pub async fn flush_infos(&mut self, infos: &[StoreInfo]) -> Result<(), HypercoreError> { + pub(crate) async fn flush_infos(&mut self, infos: &[StoreInfo]) -> Result<(), HypercoreError> { if infos.is_empty() { return Ok(()); } @@ -273,6 +274,7 @@ where impl Storage { /// New storage backed by a `RandomAccessMemory` instance. + #[instrument(err)] pub async fn new_memory() -> Result { let create = |_| async { Ok(RandomAccessMemory::default()) }.boxed(); // No reason to overwrite, as this is a new memory segment @@ -283,6 +285,7 @@ impl Storage { #[cfg(not(target_arch = "wasm32"))] impl Storage { /// New storage backed by a `RandomAccessDisk` instance. + #[instrument(err)] pub async fn new_disk(dir: &PathBuf, overwrite: bool) -> Result { let storage = |store: Store| { let name = match store { diff --git a/tests/core.rs b/tests/core.rs index 1759620c..db13563c 100644 --- a/tests/core.rs +++ b/tests/core.rs @@ -3,18 +3,32 @@ pub mod common; use anyhow::Result; use common::get_test_key_pair; use hypercore::{Builder, Storage}; +use test_log::test; -#[async_std::test] +#[test(async_std::test)] async fn hypercore_new() -> Result<()> { let storage = Storage::new_memory().await?; let _hypercore = Builder::new(storage).build(); Ok(()) } -#[async_std::test] +#[test(async_std::test)] async fn hypercore_new_with_key_pair() -> Result<()> { let storage = Storage::new_memory().await?; let key_pair = get_test_key_pair(); let _hypercore = Builder::new(storage).set_key_pair(key_pair).build().await?; Ok(()) } + +#[test(async_std::test)] +async fn hypercore_open_with_key_pair_error() -> Result<()> { + let storage = Storage::new_memory().await?; + let key_pair = get_test_key_pair(); + assert!(Builder::new(storage) + .set_key_pair(key_pair) + .set_open(true) + .build() + .await + .is_err()); + Ok(()) +} diff --git a/tests/js_interop.rs b/tests/js_interop.rs index e65a9530..d48b2542 100644 --- a/tests/js_interop.rs +++ b/tests/js_interop.rs @@ -7,6 +7,7 @@ use common::{create_hypercore_hash, get_test_key_pair}; use hypercore::{Builder, Hypercore, Storage}; use js::{cleanup, install, js_run_step, prepare_test_set}; use random_access_disk::RandomAccessDisk; +use test_log::test; #[cfg(feature = "async-std")] use async_std::test as async_test; @@ -25,7 +26,7 @@ fn init() { }); } -#[async_test] +#[test(async_test)] #[cfg_attr(not(feature = "js_interop_tests"), ignore)] async fn js_interop_js_first() -> Result<()> { init(); @@ -44,7 +45,7 @@ async fn js_interop_js_first() -> Result<()> { Ok(()) } -#[async_test] +#[test(async_test)] #[cfg_attr(not(feature = "js_interop_tests"), ignore)] async fn js_interop_rs_first() -> Result<()> { init(); From 8acc5b6e09a9ddb799796701eecb02df3baf3032 Mon Sep 17 00:00:00 2001 From: Timo Tiuraniemi Date: Mon, 27 Mar 2023 13:07:29 +0300 Subject: [PATCH 111/157] Fix bench, remove unnecessary set_ from builder --- Cargo.toml | 2 +- src/builder.rs | 6 +++--- src/storage/mod.rs | 2 +- tests/core.rs | 6 +++--- tests/js_interop.rs | 4 ++-- 5 files changed, 10 insertions(+), 10 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 98f905f6..f0419266 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "hypercore" -version = "0.11.1-beta.10" +version = "0.12.0-alpha.0" license = "MIT OR Apache-2.0" description = "Secure, distributed, append-only log" documentation = "https://docs.rs/hypercore" diff --git a/src/builder.rs b/src/builder.rs index e59e9708..b5f03466 100644 --- a/src/builder.rs +++ b/src/builder.rs @@ -1,6 +1,6 @@ use random_access_storage::RandomAccess; use std::fmt::Debug; -use tracing::{debug, instrument}; +use tracing::instrument; use crate::{Hypercore, HypercoreError, PartialKeypair, Storage}; @@ -48,13 +48,13 @@ where } /// Set key pair. - pub fn set_key_pair(mut self, key_pair: PartialKeypair) -> Self { + pub fn key_pair(mut self, key_pair: PartialKeypair) -> Self { self.0.key_pair = Some(key_pair); self } /// Set open. - pub fn set_open(mut self, open: bool) -> Self { + pub fn open(mut self, open: bool) -> Self { self.0.open = open; self } diff --git a/src/storage/mod.rs b/src/storage/mod.rs index 67727731..971ad675 100644 --- a/src/storage/mod.rs +++ b/src/storage/mod.rs @@ -83,7 +83,7 @@ where T: RandomAccess + Debug + Send, { /// Create a new instance. Takes a callback to create new storage instances and overwrite flag. - pub(crate) async fn open(create: Cb, overwrite: bool) -> Result + pub async fn open(create: Cb, overwrite: bool) -> Result where Cb: Fn( Store, diff --git a/tests/core.rs b/tests/core.rs index db13563c..be8389ef 100644 --- a/tests/core.rs +++ b/tests/core.rs @@ -16,7 +16,7 @@ async fn hypercore_new() -> Result<()> { async fn hypercore_new_with_key_pair() -> Result<()> { let storage = Storage::new_memory().await?; let key_pair = get_test_key_pair(); - let _hypercore = Builder::new(storage).set_key_pair(key_pair).build().await?; + let _hypercore = Builder::new(storage).key_pair(key_pair).build().await?; Ok(()) } @@ -25,8 +25,8 @@ async fn hypercore_open_with_key_pair_error() -> Result<()> { let storage = Storage::new_memory().await?; let key_pair = get_test_key_pair(); assert!(Builder::new(storage) - .set_key_pair(key_pair) - .set_open(true) + .key_pair(key_pair) + .open(true) .build() .await .is_err()); diff --git a/tests/js_interop.rs b/tests/js_interop.rs index d48b2542..fc9a8f7b 100644 --- a/tests/js_interop.rs +++ b/tests/js_interop.rs @@ -140,13 +140,13 @@ async fn create_hypercore(work_dir: &str) -> Result> let path = Path::new(work_dir).to_owned(); let key_pair = get_test_key_pair(); let storage = Storage::new_disk(&path, true).await?; - Ok(Builder::new(storage).set_key_pair(key_pair).build().await?) + Ok(Builder::new(storage).key_pair(key_pair).build().await?) } async fn open_hypercore(work_dir: &str) -> Result> { let path = Path::new(work_dir).to_owned(); let storage = Storage::new_disk(&path, false).await?; - Ok(Builder::new(storage).set_open(true).build().await?) + Ok(Builder::new(storage).open(true).build().await?) } fn step_0_hash() -> common::HypercoreHash { From 13672c7e2f80a6a6ff15f0cfc193d8dd688c518b Mon Sep 17 00:00:00 2001 From: Timo Tiuraniemi Date: Tue, 28 Mar 2023 08:47:33 +0300 Subject: [PATCH 112/157] Fix unflushed nodes never emptying. This had a positive performance impact by turning the unflushed nodes array into a in-memory cache, but it's also an uncontrolled memory leak. --- src/tree/merkle_tree.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/tree/merkle_tree.rs b/src/tree/merkle_tree.rs index 17342c2f..3e32dee9 100644 --- a/src/tree/merkle_tree.rs +++ b/src/tree/merkle_tree.rs @@ -650,7 +650,7 @@ impl MerkleTree { pub fn flush_nodes(&mut self) -> Vec { let mut infos_to_flush: Vec = Vec::with_capacity(self.unflushed.len()); - for node in self.unflushed.values() { + for (_, node) in self.unflushed.drain() { let (mut state, mut buffer) = State::new_with_size(40); state.encode_u64(node.length, &mut buffer); state.encode_fixed_32(&node.hash, &mut buffer); From a4a3beb3f0ce4f6179f1f700771e7415f3aa39f0 Mon Sep 17 00:00:00 2001 From: Timo Tiuraniemi Date: Tue, 28 Mar 2023 10:08:14 +0300 Subject: [PATCH 113/157] Fix byte range in some situations needing more than two calls --- src/core.rs | 64 +++++++++++++++++++++++------------------------------ 1 file changed, 28 insertions(+), 36 deletions(-) diff --git a/src/core.rs b/src/core.rs index 9784ffba..0cbf00c4 100644 --- a/src/core.rs +++ b/src/core.rs @@ -301,24 +301,7 @@ where return Ok(None); } - // TODO: Generalize Either response stack - let byte_range = match self.tree.byte_range(index, None)? { - Either::Right(value) => value, - Either::Left(instructions) => { - let infos = self.storage.read_infos(&instructions).await?; - match self.tree.byte_range(index, Some(&infos))? { - Either::Right(value) => value, - Either::Left(_) => { - return Err(HypercoreError::InvalidOperation { - context: format!( - "Could not read byte range at index {} from tree", - index - ), - }); - } - } - } - }; + let byte_range = self.byte_range(index, None).await?; // TODO: Generalize Either response stack let data = match self.block_store.read(&byte_range, None) { @@ -389,24 +372,8 @@ where }; // Find byte range for last value - let last_byte_range = match self.tree.byte_range(end - 1, Some(&infos))? { - Either::Right(value) => value, - Either::Left(instructions) => { - let new_infos = self.storage.read_infos_to_vec(&instructions).await?; - infos.extend(new_infos); - match self.tree.byte_range(end - 1, Some(&infos))? { - Either::Right(value) => value, - Either::Left(_) => { - return Err(HypercoreError::InvalidOperation { - context: format!( - "Could not read byte range for index {} from tree", - end - 1 - ), - }); - } - } - } - }; + let last_byte_range = self.byte_range(end - 1, Some(&infos)).await?; + let clear_length = (last_byte_range.index + last_byte_range.length) - clear_offset; // Clear blocks @@ -558,6 +525,31 @@ where } } + async fn byte_range( + &mut self, + index: u64, + initial_infos: Option<&[StoreInfo]>, + ) -> Result { + match self.tree.byte_range(index, initial_infos)? { + Either::Right(value) => return Ok(value), + Either::Left(instructions) => { + let mut instructions = instructions; + let mut infos: Vec = vec![]; + loop { + infos.extend(self.storage.read_infos_to_vec(&instructions).await?); + match self.tree.byte_range(index, Some(&infos))? { + Either::Right(value) => { + return Ok(value); + } + Either::Left(new_instructions) => { + instructions = new_instructions; + } + } + } + } + } + } + async fn create_valueless_proof( &mut self, block: Option, From ccd62f83104af25d9a025890df82b48bb3d091e0 Mon Sep 17 00:00:00 2001 From: Timo Tiuraniemi Date: Tue, 28 Mar 2023 10:10:14 +0300 Subject: [PATCH 114/157] Add moka cache for nodes Shows roughly 20% improvement in read benchmark for disk (benchmarks were also added in this commit). --- Cargo.toml | 6 ++ benches/disk.rs | 101 +++++++++++++++++++++++++++++++ benches/memory.rs | 58 +++++++++++++++--- src/builder.rs | 111 +++++++++++++++++----------------- src/common/cache.rs | 58 ++++++++++++++++++ src/common/mod.rs | 2 + src/core.rs | 110 ++++++++++++++++++++++------------ src/lib.rs | 4 +- src/tree/merkle_tree.rs | 129 ++++++++++++++++++++++++---------------- tests/core.rs | 11 ++-- tests/js_interop.rs | 9 ++- tests/model.rs | 4 +- 12 files changed, 441 insertions(+), 162 deletions(-) create mode 100644 benches/disk.rs create mode 100644 src/common/cache.rs diff --git a/Cargo.toml b/Cargo.toml index f0419266..9d812b7e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -44,6 +44,7 @@ bitfield-rle = "0.2.0" futures = "0.3.4" crc32fast = "1.3.2" intmap = "2.0.0" +moka = { version = "0.10.0", optional = true } [target.'cfg(not(target_arch = "wasm32"))'.dependencies] random-access-disk = { git = "https://github.com/ttiurani/random-access-disk", branch = "thiserror" } @@ -65,6 +66,7 @@ tracing-subscriber = { version = "0.3.16", features = ["env-filter", "fmt"] } default = ["async-std"] tokio = ["random-access-disk/tokio"] async-std = ["random-access-disk/async-std"] +cache = ["moka"] # Used only in interoperability tests under tests/js-interop which use the javascript version of hypercore # to verify that this crate works. To run them, use: # cargo test --features js-interop-tests @@ -73,3 +75,7 @@ js_interop_tests = [] [[bench]] name = "memory" harness = false + +[[bench]] +name = "disk" +harness = false diff --git a/benches/disk.rs b/benches/disk.rs new file mode 100644 index 00000000..66aa41d3 --- /dev/null +++ b/benches/disk.rs @@ -0,0 +1,101 @@ +use std::time::{Duration, Instant}; + +use criterion::async_executor::AsyncStdExecutor; +use criterion::{black_box, criterion_group, criterion_main, Criterion}; +use hypercore::{Hypercore, HypercoreBuilder, HypercoreError, Storage}; +use random_access_disk::RandomAccessDisk; +use tempfile::Builder as TempfileBuilder; + +#[cfg(feature = "cache")] +async fn create_hypercore(name: &str) -> Result, HypercoreError> { + let dir = TempfileBuilder::new() + .prefix(name) + .tempdir() + .unwrap() + .into_path(); + let storage = Storage::new_disk(&dir, true).await?; + HypercoreBuilder::new(storage) + .node_cache_options(hypercore::CacheOptionsBuilder::new()) + .build() + .await +} + +#[cfg(not(feature = "cache"))] +async fn create_hypercore(name: &str) -> Result, HypercoreError> { + let dir = TempfileBuilder::new() + .prefix(name) + .tempdir() + .unwrap() + .into_path(); + let storage = Storage::new_disk(&dir, true).await?; + HypercoreBuilder::new(storage).build().await +} + +fn create_disk(c: &mut Criterion) { + let mut group = c.benchmark_group("slow_call"); + group.measurement_time(Duration::from_secs(20)); + group.bench_function("create_disk", move |b| { + b.to_async(AsyncStdExecutor) + .iter(|| create_hypercore("create")); + }); +} + +fn write_disk(c: &mut Criterion) { + let mut group = c.benchmark_group("slow_call"); + group.measurement_time(Duration::from_secs(20)); + group.bench_function("write_disk", move |b| { + b.to_async(AsyncStdExecutor) + .iter_custom(|iters| async move { + let mut hypercore = create_hypercore("write").await.unwrap(); + let data = Vec::from("hello"); + let start = Instant::now(); + for _ in 0..iters { + black_box(hypercore.append(&data).await.unwrap()); + } + start.elapsed() + }); + }); +} + +fn read_disk(c: &mut Criterion) { + let mut group = c.benchmark_group("slow_call"); + group.measurement_time(Duration::from_secs(20)); + group.bench_function("read_disk", move |b| { + b.to_async(AsyncStdExecutor) + .iter_custom(|iters| async move { + let mut hypercore = create_hypercore("read").await.unwrap(); + let data = Vec::from("hello"); + for _ in 0..iters { + hypercore.append(&data).await.unwrap(); + } + let start = Instant::now(); + for i in 0..iters { + black_box(hypercore.get(i).await.unwrap()); + } + start.elapsed() + }); + }); +} + +fn clear_disk(c: &mut Criterion) { + let mut group = c.benchmark_group("slow_call"); + group.measurement_time(Duration::from_secs(20)); + group.bench_function("clear_disk", move |b| { + b.to_async(AsyncStdExecutor) + .iter_custom(|iters| async move { + let mut hypercore = create_hypercore("clear").await.unwrap(); + let data = Vec::from("hello"); + for _ in 0..iters { + hypercore.append(&data).await.unwrap(); + } + let start = Instant::now(); + for i in 0..iters { + black_box(hypercore.clear(i, 1).await.unwrap()); + } + start.elapsed() + }); + }); +} + +criterion_group!(benches, create_disk, write_disk, read_disk, clear_disk); +criterion_main!(benches); diff --git a/benches/memory.rs b/benches/memory.rs index cffcbe19..ee26f9b9 100644 --- a/benches/memory.rs +++ b/benches/memory.rs @@ -2,9 +2,10 @@ use std::time::Instant; use criterion::async_executor::AsyncStdExecutor; use criterion::{black_box, criterion_group, criterion_main, Criterion}; -use hypercore::{Builder, Hypercore, HypercoreError, Storage}; +use hypercore::{Hypercore, HypercoreBuilder, HypercoreError, Storage}; use random_access_memory::RandomAccessMemory; +#[cfg(feature = "cache")] async fn create_hypercore( page_size: usize, ) -> Result, HypercoreError> { @@ -13,17 +14,32 @@ async fn create_hypercore( false, ) .await?; - Builder::new(storage).build().await + HypercoreBuilder::new(storage) + .node_cache_options(hypercore::CacheOptionsBuilder::new()) + .build() + .await } -fn create(c: &mut Criterion) { - c.bench_function("create", move |b| { +#[cfg(not(feature = "cache"))] +async fn create_hypercore( + page_size: usize, +) -> Result, HypercoreError> { + let storage = Storage::open( + |_| Box::pin(async move { Ok(RandomAccessMemory::new(page_size)) }), + false, + ) + .await?; + HypercoreBuilder::new(storage).build().await +} + +fn create_memory(c: &mut Criterion) { + c.bench_function("create_memory", move |b| { b.to_async(AsyncStdExecutor).iter(|| create_hypercore(1024)); }); } -fn write(c: &mut Criterion) { - c.bench_function("write", move |b| { +fn write_memory(c: &mut Criterion) { + c.bench_function("write_memory", move |b| { b.to_async(AsyncStdExecutor) .iter_custom(|iters| async move { let mut hypercore = create_hypercore(1024).await.unwrap(); @@ -37,8 +53,8 @@ fn write(c: &mut Criterion) { }); } -fn read(c: &mut Criterion) { - c.bench_function("read", move |b| { +fn read_memory(c: &mut Criterion) { + c.bench_function("read_memory", move |b| { b.to_async(AsyncStdExecutor) .iter_custom(|iters| async move { let mut hypercore = create_hypercore(1024).await.unwrap(); @@ -55,5 +71,29 @@ fn read(c: &mut Criterion) { }); } -criterion_group!(benches, create, write, read); +fn clear_memory(c: &mut Criterion) { + c.bench_function("clear_memory", move |b| { + b.to_async(AsyncStdExecutor) + .iter_custom(|iters| async move { + let mut hypercore = create_hypercore(1024).await.unwrap(); + let data = Vec::from("hello"); + for _ in 0..iters { + hypercore.append(&data).await.unwrap(); + } + let start = Instant::now(); + for i in 0..iters { + black_box(hypercore.clear(i, 1).await.unwrap()); + } + start.elapsed() + }); + }); +} + +criterion_group!( + benches, + create_memory, + write_memory, + read_memory, + clear_memory +); criterion_main!(benches); diff --git a/src/builder.rs b/src/builder.rs index b5f03466..2b7a5036 100644 --- a/src/builder.rs +++ b/src/builder.rs @@ -1,88 +1,93 @@ use random_access_storage::RandomAccess; use std::fmt::Debug; +#[cfg(feature = "cache")] +use std::time::Duration; use tracing::instrument; -use crate::{Hypercore, HypercoreError, PartialKeypair, Storage}; +#[cfg(feature = "cache")] +use crate::common::cache::CacheOptions; +use crate::{core::HypercoreOptions, Hypercore, HypercoreError, PartialKeypair, Storage}; -/// Options for a Hypercore instance. +/// Build CacheOptions. +#[cfg(feature = "cache")] #[derive(Debug)] -pub struct Options -where - T: RandomAccess + Debug + Send, -{ - /// Existing key pair to use - pub key_pair: Option, - /// Storage - pub storage: Option>, - /// Whether or not to open existing or create new - pub open: bool, -} +pub struct CacheOptionsBuilder(CacheOptions); -impl Options -where - T: RandomAccess + Debug + Send, -{ - /// Create with default options. - pub fn new(storage: Storage) -> Self { - Self { - storage: Some(storage), - key_pair: None, - open: false, - } +#[cfg(feature = "cache")] +impl CacheOptionsBuilder { + /// Create a CacheOptions builder with default options + pub fn new() -> Self { + Self(CacheOptions::new()) + } + + /// Set cache time to live. + pub fn time_to_live(mut self, time_to_live: Duration) -> Self { + self.0.time_to_live = Some(time_to_live); + self + } + + /// Set cache time to idle. + pub fn time_to_idle(mut self, time_to_idle: Duration) -> Self { + self.0.time_to_idle = Some(time_to_idle); + self + } + + /// Set cache max capacity in bytes. + pub fn max_capacity(mut self, max_capacity: u64) -> Self { + self.0.max_capacity = Some(max_capacity); + self + } + + /// Build new cache options. + pub(crate) fn build(self) -> CacheOptions { + self.0 } } /// Build a Hypercore instance with options. #[derive(Debug)] -pub struct Builder(Options) +pub struct HypercoreBuilder where - T: RandomAccess + Debug + Send; + T: RandomAccess + Debug + Send, +{ + storage: Storage, + options: HypercoreOptions, +} -impl Builder +impl HypercoreBuilder where T: RandomAccess + Debug + Send, { /// Create a hypercore builder with a given storage pub fn new(storage: Storage) -> Self { - Self(Options::new(storage)) + Self { + storage, + options: HypercoreOptions::new(), + } } /// Set key pair. pub fn key_pair(mut self, key_pair: PartialKeypair) -> Self { - self.0.key_pair = Some(key_pair); + self.options.key_pair = Some(key_pair); self } /// Set open. pub fn open(mut self, open: bool) -> Self { - self.0.open = open; + self.options.open = open; + self + } + + /// Set node cache options. + #[cfg(feature = "cache")] + pub fn node_cache_options(mut self, builder: CacheOptionsBuilder) -> Self { + self.options.node_cache_options = Some(builder.build()); self } /// Build a new Hypercore. #[instrument(err, skip_all)] - pub async fn build(mut self) -> Result, HypercoreError> { - let storage = self - .0 - .storage - .take() - .ok_or_else(|| HypercoreError::BadArgument { - context: "Storage must be provided when building a hypercore".to_string(), - })?; - if self.0.open { - if self.0.key_pair.is_some() { - return Err(HypercoreError::BadArgument { - context: "Key pair can not be used when building an openable hypercore" - .to_string(), - }); - } - Hypercore::open(storage).await - } else { - if let Some(key_pair) = self.0.key_pair.take() { - Hypercore::new_with_key_pair(storage, key_pair).await - } else { - Hypercore::new(storage).await - } - } + pub async fn build(self) -> Result, HypercoreError> { + Hypercore::new(self.storage, self.options).await } } diff --git a/src/common/cache.rs b/src/common/cache.rs new file mode 100644 index 00000000..fc6a4961 --- /dev/null +++ b/src/common/cache.rs @@ -0,0 +1,58 @@ +use moka::sync::Cache; +use std::time::Duration; + +use crate::Node; + +// Default to 1 year of cache +const DEFAULT_CACHE_TTL_SEC: u64 = 31556952; +const DEFAULT_CACHE_TTI_SEC: u64 = 31556952; +// Default to 100kb of node cache +const DEFAULT_CACHE_MAX_SIZE: u64 = 100000; +const NODE_WEIGHT: u32 = + // Byte size of a Node based on the fields. + 3 * 8 + 32 + 4 + + // Then 8 for key and guesstimate 8 bytes of overhead. + 8 + 8; + +#[derive(Debug, Clone)] +pub(crate) struct CacheOptions { + pub(crate) time_to_live: Option, + pub(crate) time_to_idle: Option, + pub(crate) max_capacity: Option, +} + +impl CacheOptions { + pub(crate) fn new() -> Self { + Self { + time_to_live: None, + time_to_idle: None, + max_capacity: None, + } + } + + pub(crate) fn to_node_cache(&self, initial_nodes: Vec) -> Cache { + let cache = if self.time_to_live.is_some() || self.time_to_idle.is_some() { + Cache::builder() + .time_to_live( + self.time_to_live + .unwrap_or_else(|| Duration::from_secs(DEFAULT_CACHE_TTL_SEC)), + ) + .time_to_idle( + self.time_to_idle + .unwrap_or_else(|| Duration::from_secs(DEFAULT_CACHE_TTI_SEC)), + ) + .max_capacity(self.max_capacity.unwrap_or(DEFAULT_CACHE_MAX_SIZE)) + .weigher(|_, _| NODE_WEIGHT) + .build() + } else { + Cache::builder() + .max_capacity(self.max_capacity.unwrap_or(DEFAULT_CACHE_MAX_SIZE)) + .weigher(|_, _| NODE_WEIGHT) + .build() + }; + for node in initial_nodes { + cache.insert(node.index, node); + } + cache + } +} diff --git a/src/common/mod.rs b/src/common/mod.rs index e13812ca..b43722de 100644 --- a/src/common/mod.rs +++ b/src/common/mod.rs @@ -1,3 +1,5 @@ +#[cfg(feature = "cache")] +pub(crate) mod cache; mod error; mod node; mod peer; diff --git a/src/core.rs b/src/core.rs index 0cbf00c4..fed16223 100644 --- a/src/core.rs +++ b/src/core.rs @@ -1,8 +1,16 @@ //! Hypercore's main abstraction. Exposes an append-only, secure log structure. +use ed25519_dalek::Signature; +use futures::future::Either; +use random_access_storage::RandomAccess; +use std::convert::TryFrom; +use std::fmt::Debug; +use tracing::instrument; +#[cfg(feature = "cache")] +use crate::common::cache::CacheOptions; use crate::{ bitfield::Bitfield, - common::{BitfieldUpdate, HypercoreError, Proof, StoreInfo, ValuelessProof}, + common::{BitfieldUpdate, HypercoreError, NodeByteRange, Proof, StoreInfo, ValuelessProof}, crypto::generate_keypair, data::BlockStore, oplog::{Header, Oplog, MAX_OPLOG_ENTRIES_BYTE_SIZE}, @@ -10,12 +18,25 @@ use crate::{ tree::{MerkleTree, MerkleTreeChangeset}, RequestBlock, RequestSeek, RequestUpgrade, }; -use ed25519_dalek::Signature; -use futures::future::Either; -use random_access_storage::RandomAccess; -use std::convert::TryFrom; -use std::fmt::Debug; -use tracing::instrument; + +#[derive(Debug)] +pub(crate) struct HypercoreOptions { + pub(crate) key_pair: Option, + pub(crate) open: bool, + #[cfg(feature = "cache")] + pub(crate) node_cache_options: Option, +} + +impl HypercoreOptions { + pub(crate) fn new() -> Self { + Self { + key_pair: None, + open: false, + #[cfg(feature = "cache")] + node_cache_options: None, + } + } +} /// Hypercore is an append-only log structure. #[derive(Debug)] @@ -54,37 +75,29 @@ impl Hypercore where T: RandomAccess + Debug + Send, { - /// Creates new hypercore using given storage with random key pair - pub(crate) async fn new(storage: Storage) -> Result, HypercoreError> { - let key_pair = generate_keypair(); - Hypercore::new_with_key_pair( - storage, - PartialKeypair { - public: key_pair.public, - secret: Some(key_pair.secret), - }, - ) - .await - } - - /// Creates new hypercore with given storage and (partial) key pair - pub(crate) async fn new_with_key_pair( - storage: Storage, - key_pair: PartialKeypair, - ) -> Result, HypercoreError> { - Hypercore::resume(storage, Some(key_pair)).await - } - - /// Opens an existing hypercore in given storage. - pub(crate) async fn open(storage: Storage) -> Result, HypercoreError> { - Hypercore::resume(storage, None).await - } - - /// Creates/opens a hypercore with given storage and potentially a key pair - async fn resume( + /// Creates/opens new hypercore using given storage and options + pub(crate) async fn new( mut storage: Storage, - key_pair: Option, + mut options: HypercoreOptions, ) -> Result, HypercoreError> { + let key_pair: Option = if options.open { + if options.key_pair.is_some() { + return Err(HypercoreError::BadArgument { + context: "Key pair can not be used when building an openable hypercore" + .to_string(), + }); + } + None + } else { + Some(options.key_pair.take().unwrap_or_else(|| { + let key_pair = generate_keypair(); + PartialKeypair { + public: key_pair.public, + secret: Some(key_pair.secret), + } + })) + }; + // Open/create oplog let mut oplog_open_outcome = match Oplog::open(&key_pair, None)? { Either::Right(value) => value, @@ -105,11 +118,21 @@ where .await?; // Open/create tree - let mut tree = match MerkleTree::open(&oplog_open_outcome.header.tree, None)? { + let mut tree = match MerkleTree::open( + &oplog_open_outcome.header.tree, + None, + #[cfg(feature = "cache")] + &options.node_cache_options, + )? { Either::Right(value) => value, Either::Left(instructions) => { let infos = storage.read_infos(&instructions).await?; - match MerkleTree::open(&oplog_open_outcome.header.tree, Some(&infos))? { + match MerkleTree::open( + &oplog_open_outcome.header.tree, + Some(&infos), + #[cfg(feature = "cache")] + &options.node_cache_options, + )? { Either::Right(value) => value, Either::Left(_) => { return Err(HypercoreError::InvalidOperation { @@ -1046,7 +1069,16 @@ mod tests { key_pair: PartialKeypair, ) -> Result, HypercoreError> { let storage = Storage::new_memory().await?; - let mut hypercore = Hypercore::new_with_key_pair(storage, key_pair).await?; + let mut hypercore = Hypercore::new( + storage, + HypercoreOptions { + key_pair: Some(key_pair), + open: false, + #[cfg(feature = "cache")] + node_cache_options: None, + }, + ) + .await?; for i in 0..length { hypercore.append(format!("#{}", i).as_bytes()).await?; } diff --git a/src/lib.rs b/src/lib.rs index 5852fa64..28eecf9e 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -38,7 +38,9 @@ mod oplog; mod storage; mod tree; -pub use crate::builder::Builder; +#[cfg(feature = "cache")] +pub use crate::builder::CacheOptionsBuilder; +pub use crate::builder::HypercoreBuilder; pub use crate::common::{ DataBlock, DataHash, DataSeek, DataUpgrade, HypercoreError, Node, Proof, RequestBlock, RequestSeek, RequestUpgrade, Store, diff --git a/src/tree/merkle_tree.rs b/src/tree/merkle_tree.rs index 3e32dee9..3e9fca81 100644 --- a/src/tree/merkle_tree.rs +++ b/src/tree/merkle_tree.rs @@ -2,8 +2,12 @@ use compact_encoding::State; use ed25519_dalek::Signature; use futures::future::Either; use intmap::IntMap; +#[cfg(feature = "cache")] +use moka::sync::Cache; use std::convert::TryFrom; +#[cfg(feature = "cache")] +use crate::common::cache::CacheOptions; use crate::common::{HypercoreError, NodeByteRange, Proof, ValuelessProof}; use crate::crypto::{Hash, PublicKey}; use crate::oplog::HeaderTree; @@ -29,15 +33,18 @@ pub struct MerkleTree { unflushed: IntMap, truncated: bool, truncate_to: u64, + #[cfg(feature = "cache")] + node_cache: Option>, } const NODE_SIZE: u64 = 40; impl MerkleTree { /// Opens MerkleTree, based on read infos. - pub fn open( + pub(crate) fn open( header_tree: &HeaderTree, infos: Option<&[StoreInfo]>, + #[cfg(feature = "cache")] node_cache_options: &Option, ) -> Result, Self>, HypercoreError> { match infos { None => { @@ -86,16 +93,22 @@ impl MerkleTree { length = length / 2; } let signature: Option = if header_tree.signature.len() > 0 { - Some(Signature::try_from(&*header_tree.signature).map_err(|err| { - HypercoreError::InvalidSignature { - context: "Could not parse signature".to_string(), - } - })?) + Some( + Signature::try_from(&*header_tree.signature).map_err(|_err| { + HypercoreError::InvalidSignature { + context: "Could not parse signature".to_string(), + } + })?, + ) } else { None }; Ok(Either::Right(Self { + #[cfg(feature = "cache")] + node_cache: node_cache_options + .as_ref() + .map(|opts| opts.to_node_cache(roots.clone())), roots, length, byte_length, @@ -112,12 +125,12 @@ impl MerkleTree { /// Initialize a changeset for this tree. /// This is called batch() in Javascript, see: /// https://github.com/hypercore-protocol/hypercore/blob/master/lib/merkle-tree.js - pub fn changeset(&self) -> MerkleTreeChangeset { + pub(crate) fn changeset(&self) -> MerkleTreeChangeset { MerkleTreeChangeset::new(self.length, self.byte_length, self.fork, self.roots.clone()) } /// Commit a created changeset to the tree. - pub fn commit(&mut self, changeset: MerkleTreeChangeset) -> Result<(), HypercoreError> { + pub(crate) fn commit(&mut self, changeset: MerkleTreeChangeset) -> Result<(), HypercoreError> { if !self.commitable(&changeset) { return Err(HypercoreError::InvalidOperation { context: "Tree was modified during changeset, refusing to commit".to_string(), @@ -142,7 +155,7 @@ impl MerkleTree { } /// Flush committed made changes to the tree - pub fn flush(&mut self) -> Box<[StoreInfo]> { + pub(crate) fn flush(&mut self) -> Box<[StoreInfo]> { let mut infos_to_flush: Vec = Vec::new(); if self.truncated { infos_to_flush.extend(self.flush_truncation()); @@ -153,13 +166,13 @@ impl MerkleTree { /// Get storage byte range of given hypercore index pub fn byte_range( - &self, + &mut self, hypercore_index: u64, infos: Option<&[StoreInfo]>, ) -> Result, NodeByteRange>, HypercoreError> { let index = self.validate_hypercore_index(hypercore_index)?; // Get nodes out of incoming infos - let nodes: IntMap> = infos_to_nodes(infos); + let nodes: IntMap> = self.infos_to_nodes(infos); // Start with getting the requested node, which will get the length // of the byte range @@ -200,8 +213,8 @@ impl MerkleTree { } /// Get the byte offset given hypercore index - pub fn byte_offset( - &self, + pub(crate) fn byte_offset( + &mut self, hypercore_index: u64, infos: Option<&[StoreInfo]>, ) -> Result, u64>, HypercoreError> { @@ -210,8 +223,8 @@ impl MerkleTree { } /// Get the byte offset of hypercore index in a changeset - pub fn byte_offset_in_changeset( - &self, + pub(crate) fn byte_offset_in_changeset( + &mut self, hypercore_index: u64, changeset: &MerkleTreeChangeset, infos: Option<&[StoreInfo]>, @@ -259,11 +272,11 @@ impl MerkleTree { } } - pub fn add_node(&mut self, node: Node) { + pub(crate) fn add_node(&mut self, node: Node) { self.unflushed.insert(node.index, node); } - pub fn truncate( + pub(crate) fn truncate( &mut self, length: u64, fork: u64, @@ -272,7 +285,7 @@ impl MerkleTree { let head = length * 2; let mut full_roots = vec![]; flat_tree::full_roots(head, &mut full_roots); - let nodes: IntMap> = infos_to_nodes(infos); + let nodes: IntMap> = self.infos_to_nodes(infos); let mut changeset = self.changeset(); let mut instructions: Vec = Vec::new(); @@ -318,15 +331,15 @@ impl MerkleTree { /// TODO: This is now just a clone of javascript's /// https://github.com/hypercore-protocol/hypercore/blob/7e30a0fe353c70ada105840ec1ead6627ff521e7/lib/merkle-tree.js#L604 /// The implementation should be rewritten to make it clearer. - pub fn create_valueless_proof( - &self, + pub(crate) fn create_valueless_proof( + &mut self, block: Option<&RequestBlock>, hash: Option<&RequestBlock>, seek: Option<&RequestSeek>, upgrade: Option<&RequestUpgrade>, infos: Option<&[StoreInfo]>, ) -> Result, ValuelessProof>, HypercoreError> { - let nodes: IntMap> = infos_to_nodes(infos); + let nodes: IntMap> = self.infos_to_nodes(infos); let mut instructions: Vec = Vec::new(); let fork = self.fork; let signature = self.signature; @@ -499,13 +512,13 @@ impl MerkleTree { } /// Verifies a proof received from a peer. - pub fn verify_proof( - &self, + pub(crate) fn verify_proof( + &mut self, proof: &Proof, public_key: &PublicKey, infos: Option<&[StoreInfo]>, ) -> Result, MerkleTreeChangeset>, HypercoreError> { - let nodes: IntMap> = infos_to_nodes(infos); + let nodes: IntMap> = self.infos_to_nodes(infos); let mut instructions: Vec = Vec::new(); let mut changeset = self.changeset(); @@ -556,8 +569,8 @@ impl MerkleTree { } /// Attempts to get missing nodes from given index. NB: must be called in a loop. - pub fn missing_nodes( - &self, + pub(crate) fn missing_nodes( + &mut self, index: u64, infos: Option<&[StoreInfo]>, ) -> Result, u64>, HypercoreError> { @@ -570,7 +583,7 @@ impl MerkleTree { return Ok(Either::Right(0)); } - let nodes: IntMap> = infos_to_nodes(infos); + let nodes: IntMap> = self.infos_to_nodes(infos); let mut count: u64 = 0; while !iter.contains(head) { match self.optional_node(iter.index(), &nodes)? { @@ -591,7 +604,7 @@ impl MerkleTree { } /// Is the changeset commitable to given tree - pub fn commitable(&self, changeset: &MerkleTreeChangeset) -> bool { + pub(crate) fn commitable(&self, changeset: &MerkleTreeChangeset) -> bool { let correct_length: bool = if changeset.upgraded { changeset.original_tree_length == self.length } else { @@ -684,12 +697,12 @@ impl MerkleTree { } fn byte_offset_from_index( - &self, + &mut self, index: u64, infos: Option<&[StoreInfo]>, ) -> Result, u64>, HypercoreError> { // Get nodes out of incoming infos - let nodes: IntMap> = infos_to_nodes(infos); + let nodes: IntMap> = self.infos_to_nodes(infos); // Get offset let offset_result = self.byte_offset_from_nodes(index, &nodes)?; // Get offset @@ -786,7 +799,15 @@ impl MerkleTree { nodes: &IntMap>, allow_miss: bool, ) -> Result>, HypercoreError> { - // First check if unflushed already has the node + // First check the cache + #[cfg(feature = "cache")] + if let Some(node_cache) = &self.node_cache { + if let Some(node) = node_cache.get(&index) { + return Ok(Either::Right(Some(node.clone()))); + } + } + + // Then check if unflushed has the node if let Some(node) = self.unflushed.get(index) { if node.blank || (self.truncated && node.index >= 2 * self.truncate_to) { // The node is either blank or being deleted @@ -805,7 +826,7 @@ impl MerkleTree { return Ok(Either::Right(Some(node.clone()))); } - // Then check if it's already in the incoming nodes + // Then check if it's in the incoming nodes let result = nodes.get(index); if let Some(node_maybe) = result { if let Some(node) = node_maybe { @@ -1280,6 +1301,31 @@ impl MerkleTree { Ok(Either::Left(instructions)) } } + + fn infos_to_nodes(&mut self, infos: Option<&[StoreInfo]>) -> IntMap> { + match infos { + Some(infos) => { + let mut nodes: IntMap> = IntMap::with_capacity(infos.len()); + for info in infos { + let index = index_from_info(&info); + if !info.miss { + let node = node_from_bytes(&index, info.data.as_ref().unwrap()); + #[cfg(feature = "cache")] + if !node.blank { + if let Some(node_cache) = &self.node_cache { + node_cache.insert(node.index, node.clone()) + } + } + nodes.insert(index, Some(node)); + } else { + nodes.insert(index, None); + } + } + nodes + } + None => IntMap::new(), + } + } } /// Converts a hypercore index into a merkle tree index. In the flat tree @@ -1436,25 +1482,6 @@ fn node_from_bytes(index: &u64, data: &[u8]) -> Node { Node::new(*index, hash.to_vec(), len) } -fn infos_to_nodes(infos: Option<&[StoreInfo]>) -> IntMap> { - match infos { - Some(infos) => { - let mut nodes: IntMap> = IntMap::with_capacity(infos.len()); - for info in infos { - let index = index_from_info(&info); - if !info.miss { - let node = node_from_bytes(&index, info.data.as_ref().unwrap()); - nodes.insert(index, Some(node)); - } else { - nodes.insert(index, None); - } - } - nodes - } - None => IntMap::new(), - } -} - #[derive(Debug, Copy, Clone)] struct NormalizedIndexed { pub value: bool, diff --git a/tests/core.rs b/tests/core.rs index be8389ef..982d8b18 100644 --- a/tests/core.rs +++ b/tests/core.rs @@ -2,13 +2,13 @@ pub mod common; use anyhow::Result; use common::get_test_key_pair; -use hypercore::{Builder, Storage}; +use hypercore::{HypercoreBuilder, Storage}; use test_log::test; #[test(async_std::test)] async fn hypercore_new() -> Result<()> { let storage = Storage::new_memory().await?; - let _hypercore = Builder::new(storage).build(); + let _hypercore = HypercoreBuilder::new(storage).build(); Ok(()) } @@ -16,7 +16,10 @@ async fn hypercore_new() -> Result<()> { async fn hypercore_new_with_key_pair() -> Result<()> { let storage = Storage::new_memory().await?; let key_pair = get_test_key_pair(); - let _hypercore = Builder::new(storage).key_pair(key_pair).build().await?; + let _hypercore = HypercoreBuilder::new(storage) + .key_pair(key_pair) + .build() + .await?; Ok(()) } @@ -24,7 +27,7 @@ async fn hypercore_new_with_key_pair() -> Result<()> { async fn hypercore_open_with_key_pair_error() -> Result<()> { let storage = Storage::new_memory().await?; let key_pair = get_test_key_pair(); - assert!(Builder::new(storage) + assert!(HypercoreBuilder::new(storage) .key_pair(key_pair) .open(true) .build() diff --git a/tests/js_interop.rs b/tests/js_interop.rs index fc9a8f7b..c9d888a6 100644 --- a/tests/js_interop.rs +++ b/tests/js_interop.rs @@ -4,7 +4,7 @@ use std::{path::Path, sync::Once}; use anyhow::Result; use common::{create_hypercore_hash, get_test_key_pair}; -use hypercore::{Builder, Hypercore, Storage}; +use hypercore::{Hypercore, HypercoreBuilder, Storage}; use js::{cleanup, install, js_run_step, prepare_test_set}; use random_access_disk::RandomAccessDisk; use test_log::test; @@ -140,13 +140,16 @@ async fn create_hypercore(work_dir: &str) -> Result> let path = Path::new(work_dir).to_owned(); let key_pair = get_test_key_pair(); let storage = Storage::new_disk(&path, true).await?; - Ok(Builder::new(storage).key_pair(key_pair).build().await?) + Ok(HypercoreBuilder::new(storage) + .key_pair(key_pair) + .build() + .await?) } async fn open_hypercore(work_dir: &str) -> Result> { let path = Path::new(work_dir).to_owned(); let storage = Storage::new_disk(&path, false).await?; - Ok(Builder::new(storage).open(true).build().await?) + Ok(HypercoreBuilder::new(storage).open(true).build().await?) } fn step_0_hash() -> common::HypercoreHash { diff --git a/tests/model.rs b/tests/model.rs index fe6f4e68..c0b525dd 100644 --- a/tests/model.rs +++ b/tests/model.rs @@ -66,12 +66,12 @@ quickcheck! { } async fn assert_implementation_matches_model(ops: Vec) -> bool { - use hypercore::{Builder, Storage}; + use hypercore::{HypercoreBuilder, Storage}; let storage = Storage::new_memory() .await .expect("Memory storage creation should be successful"); - let mut hypercore = Builder::new(storage) + let mut hypercore = HypercoreBuilder::new(storage) .build() .await .expect("Hypercore creation should be successful"); From 51016202fc45b9c15a138073b831832c403aca45 Mon Sep 17 00:00:00 2001 From: Timo Tiuraniemi Date: Mon, 3 Apr 2023 17:26:41 +0300 Subject: [PATCH 115/157] Use error-returning compact encoding --- Cargo.toml | 2 +- src/common/error.rs | 9 ++ src/core.rs | 15 +-- src/crypto/hash.rs | 32 ++++-- src/encoding.rs | 224 +++++++++++++++++++++------------------- src/oplog/entry.rs | 117 +++++++++++---------- src/oplog/header.rs | 211 ++++++++++++++++++------------------- src/oplog/mod.rs | 101 +++++++++--------- src/tree/merkle_tree.rs | 39 ++++--- 9 files changed, 405 insertions(+), 345 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 9d812b7e..89477c32 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -27,7 +27,7 @@ byteorder = "1.3.4" ed25519-dalek = "1.0.1" thiserror = "1" tracing = "0.1" -compact-encoding = { git = "https://github.com/datrs/compact-encoding", branch = "main" } +compact-encoding = { git = "https://github.com/datrs/compact-encoding", branch = "error" } flat-tree = { git = "https://github.com/ttiurani/flat-tree", branch = "v10" } lazy_static = "1.4.0" memory-pager = "0.9.0" diff --git a/src/common/error.rs b/src/common/error.rs index 01888090..4a55209d 100644 --- a/src/common/error.rs +++ b/src/common/error.rs @@ -1,3 +1,4 @@ +use compact_encoding::EncodingError; use thiserror::Error; use crate::Store; @@ -67,3 +68,11 @@ impl From for HypercoreError { } } } + +impl From for HypercoreError { + fn from(err: EncodingError) -> Self { + Self::InvalidOperation { + context: format!("Encoding failed: {}", err), + } + } +} diff --git a/src/core.rs b/src/core.rs index fed16223..61a561ec 100644 --- a/src/core.rs +++ b/src/core.rs @@ -291,7 +291,7 @@ where Some(bitfield_update.clone()), false, &self.header, - ); + )?; self.storage.flush_infos(&outcome.infos_to_flush).await?; self.header = outcome.header; @@ -353,7 +353,7 @@ where return Ok(()); } // Write to oplog - let infos_to_flush = self.oplog.clear(start, end); + let infos_to_flush = self.oplog.clear(start, end)?; self.storage.flush_infos(&infos_to_flush).await?; // Set bitfield @@ -500,9 +500,12 @@ where }; // Append the changeset to the Oplog - let outcome = - self.oplog - .append_changeset(&changeset, bitfield_update.clone(), false, &self.header); + let outcome = self.oplog.append_changeset( + &changeset, + bitfield_update.clone(), + false, + &self.header, + )?; self.storage.flush_infos(&outcome.infos_to_flush).await?; self.header = outcome.header; @@ -651,7 +654,7 @@ where self.storage.flush_infos(&infos).await?; let infos = self.tree.flush(); self.storage.flush_infos(&infos).await?; - let infos_to_flush = self.oplog.flush(&self.header); + let infos_to_flush = self.oplog.flush(&self.header)?; self.storage.flush_infos(&infos_to_flush).await?; Ok(()) } diff --git a/src/crypto/hash.rs b/src/crypto/hash.rs index 4579ffb1..037f629e 100644 --- a/src/crypto/hash.rs +++ b/src/crypto/hash.rs @@ -106,7 +106,9 @@ impl Hash { /// Hash data pub fn data(data: &[u8]) -> Self { let (mut state, mut size) = State::new_with_size(8); - state.encode_u64(data.len() as u64, &mut size); + state + .encode_u64(data.len() as u64, &mut size) + .expect("Encoding u64 should not fail"); let mut hasher = Blake2b::new(32); hasher.update(&LEAF_TYPE); @@ -127,7 +129,9 @@ impl Hash { }; let (mut state, mut size) = State::new_with_size(8); - state.encode_u64((node1.length + node2.length) as u64, &mut size); + state + .encode_u64((node1.length + node2.length) as u64, &mut size) + .expect("Encoding u64 should not fail"); let mut hasher = Blake2b::new(32); hasher.update(&PARENT_TYPE); @@ -148,8 +152,12 @@ impl Hash { for node in roots { let node = node.as_ref(); let (mut state, mut buffer) = State::new_with_size(16); - state.encode_u64(node.index() as u64, &mut buffer); - state.encode_u64(node.len() as u64, &mut buffer); + state + .encode_u64(node.index() as u64, &mut buffer) + .expect("Encoding u64 should not fail"); + state + .encode_u64(node.len() as u64, &mut buffer) + .expect("Encoding u64 should not fail"); hasher.update(node.hash()); hasher.update(&buffer[..8]); @@ -186,10 +194,18 @@ impl DerefMut for Hash { /// See https://github.com/hypercore-protocol/hypercore/blob/70b271643c4e4b1e5ecae5bb579966dfe6361ff3/lib/caps.js#L17 pub fn signable_tree(hash: &[u8], length: u64, fork: u64) -> Box<[u8]> { let (mut state, mut buffer) = State::new_with_size(80); - state.encode_fixed_32(&TREE, &mut buffer); - state.encode_fixed_32(&hash, &mut buffer); - state.encode_u64(length, &mut buffer); - state.encode_u64(fork, &mut buffer); + state + .encode_fixed_32(&TREE, &mut buffer) + .expect("Should be able "); + state + .encode_fixed_32(&hash, &mut buffer) + .expect("Encoding fixed 32 bytes should not fail"); + state + .encode_u64(length, &mut buffer) + .expect("Encoding u64 should not fail"); + state + .encode_u64(fork, &mut buffer) + .expect("Encoding u64 should not fail"); buffer } diff --git a/src/encoding.rs b/src/encoding.rs index 1e25bd9f..b6f1ce3a 100644 --- a/src/encoding.rs +++ b/src/encoding.rs @@ -1,5 +1,5 @@ //! Hypercore-specific compact encodings -pub use compact_encoding::{CompactEncoding, State}; +pub use compact_encoding::{CompactEncoding, EncodingError, State}; use std::ops::{Deref, DerefMut}; use crate::{ @@ -48,194 +48,200 @@ impl DerefMut for HypercoreState { } impl CompactEncoding for HypercoreState { - fn preencode(&mut self, value: &Node) { - self.0.preencode(&value.index); - self.0.preencode(&value.length); - self.0.preencode_fixed_32(); + fn preencode(&mut self, value: &Node) -> Result { + self.0.preencode(&value.index)?; + self.0.preencode(&value.length)?; + self.0.preencode_fixed_32() } - fn encode(&mut self, value: &Node, buffer: &mut [u8]) { - self.0.encode(&value.index, buffer); - self.0.encode(&value.length, buffer); - self.0.encode_fixed_32(&value.hash, buffer); + fn encode(&mut self, value: &Node, buffer: &mut [u8]) -> Result { + self.0.encode(&value.index, buffer)?; + self.0.encode(&value.length, buffer)?; + self.0.encode_fixed_32(&value.hash, buffer) } - fn decode(&mut self, buffer: &[u8]) -> Node { - let index: u64 = self.0.decode(buffer); - let length: u64 = self.0.decode(buffer); - let hash: Box<[u8]> = self.0.decode_fixed_32(buffer); - Node::new(index, hash.to_vec(), length) + fn decode(&mut self, buffer: &[u8]) -> Result { + let index: u64 = self.0.decode(buffer)?; + let length: u64 = self.0.decode(buffer)?; + let hash: Box<[u8]> = self.0.decode_fixed_32(buffer)?; + Ok(Node::new(index, hash.to_vec(), length)) } } impl CompactEncoding> for HypercoreState { - fn preencode(&mut self, value: &Vec) { + fn preencode(&mut self, value: &Vec) -> Result { let len = value.len(); - self.0.preencode(&len); + self.0.preencode(&len)?; for val in value.into_iter() { - self.preencode(val); + self.preencode(val)?; } + Ok(self.end()) } - fn encode(&mut self, value: &Vec, buffer: &mut [u8]) { + fn encode(&mut self, value: &Vec, buffer: &mut [u8]) -> Result { let len = value.len(); - self.0.encode(&len, buffer); + self.0.encode(&len, buffer)?; for val in value { - self.encode(val, buffer); + self.encode(val, buffer)?; } + Ok(self.start()) } - fn decode(&mut self, buffer: &[u8]) -> Vec { - let len: usize = self.0.decode(buffer); + fn decode(&mut self, buffer: &[u8]) -> Result, EncodingError> { + let len: usize = self.0.decode(buffer)?; let mut value = Vec::with_capacity(len); for _ in 0..len { - value.push(self.decode(buffer)); + value.push(self.decode(buffer)?); } - value + Ok(value) } } impl CompactEncoding for HypercoreState { - fn preencode(&mut self, value: &RequestBlock) { - self.0.preencode(&value.index); - self.0.preencode(&value.nodes); + fn preencode(&mut self, value: &RequestBlock) -> Result { + self.0.preencode(&value.index)?; + self.0.preencode(&value.nodes) } - fn encode(&mut self, value: &RequestBlock, buffer: &mut [u8]) { - self.0.encode(&value.index, buffer); - self.0.encode(&value.nodes, buffer); + fn encode(&mut self, value: &RequestBlock, buffer: &mut [u8]) -> Result { + self.0.encode(&value.index, buffer)?; + self.0.encode(&value.nodes, buffer) } - fn decode(&mut self, buffer: &[u8]) -> RequestBlock { - let index: u64 = self.0.decode(buffer); - let nodes: u64 = self.0.decode(buffer); - RequestBlock { index, nodes } + fn decode(&mut self, buffer: &[u8]) -> Result { + let index: u64 = self.0.decode(buffer)?; + let nodes: u64 = self.0.decode(buffer)?; + Ok(RequestBlock { index, nodes }) } } impl CompactEncoding for HypercoreState { - fn preencode(&mut self, value: &RequestSeek) { - self.0.preencode(&value.bytes); + fn preencode(&mut self, value: &RequestSeek) -> Result { + self.0.preencode(&value.bytes) } - fn encode(&mut self, value: &RequestSeek, buffer: &mut [u8]) { - self.0.encode(&value.bytes, buffer); + fn encode(&mut self, value: &RequestSeek, buffer: &mut [u8]) -> Result { + self.0.encode(&value.bytes, buffer) } - fn decode(&mut self, buffer: &[u8]) -> RequestSeek { - let bytes: u64 = self.0.decode(buffer); - RequestSeek { bytes } + fn decode(&mut self, buffer: &[u8]) -> Result { + let bytes: u64 = self.0.decode(buffer)?; + Ok(RequestSeek { bytes }) } } impl CompactEncoding for HypercoreState { - fn preencode(&mut self, value: &RequestUpgrade) { - self.0.preencode(&value.start); - self.0.preencode(&value.length); + fn preencode(&mut self, value: &RequestUpgrade) -> Result { + self.0.preencode(&value.start)?; + self.0.preencode(&value.length) } - fn encode(&mut self, value: &RequestUpgrade, buffer: &mut [u8]) { - self.0.encode(&value.start, buffer); - self.0.encode(&value.length, buffer); + fn encode( + &mut self, + value: &RequestUpgrade, + buffer: &mut [u8], + ) -> Result { + self.0.encode(&value.start, buffer)?; + self.0.encode(&value.length, buffer) } - fn decode(&mut self, buffer: &[u8]) -> RequestUpgrade { - let start: u64 = self.0.decode(buffer); - let length: u64 = self.0.decode(buffer); - RequestUpgrade { start, length } + fn decode(&mut self, buffer: &[u8]) -> Result { + let start: u64 = self.0.decode(buffer)?; + let length: u64 = self.0.decode(buffer)?; + Ok(RequestUpgrade { start, length }) } } impl CompactEncoding for HypercoreState { - fn preencode(&mut self, value: &DataBlock) { - self.0.preencode(&value.index); - self.0.preencode(&value.value); - self.preencode(&value.nodes); + fn preencode(&mut self, value: &DataBlock) -> Result { + self.0.preencode(&value.index)?; + self.0.preencode(&value.value)?; + self.preencode(&value.nodes) } - fn encode(&mut self, value: &DataBlock, buffer: &mut [u8]) { - self.0.encode(&value.index, buffer); - self.0.encode(&value.value, buffer); - self.encode(&value.nodes, buffer); + fn encode(&mut self, value: &DataBlock, buffer: &mut [u8]) -> Result { + self.0.encode(&value.index, buffer)?; + self.0.encode(&value.value, buffer)?; + self.encode(&value.nodes, buffer) } - fn decode(&mut self, buffer: &[u8]) -> DataBlock { - let index: u64 = self.0.decode(buffer); - let value: Vec = self.0.decode(buffer); - let nodes: Vec = self.decode(buffer); - DataBlock { + fn decode(&mut self, buffer: &[u8]) -> Result { + let index: u64 = self.0.decode(buffer)?; + let value: Vec = self.0.decode(buffer)?; + let nodes: Vec = self.decode(buffer)?; + Ok(DataBlock { index, value, nodes, - } + }) } } impl CompactEncoding for HypercoreState { - fn preencode(&mut self, value: &DataHash) { - self.0.preencode(&value.index); - self.preencode(&value.nodes); + fn preencode(&mut self, value: &DataHash) -> Result { + self.0.preencode(&value.index)?; + self.preencode(&value.nodes) } - fn encode(&mut self, value: &DataHash, buffer: &mut [u8]) { - self.0.encode(&value.index, buffer); - self.encode(&value.nodes, buffer); + fn encode(&mut self, value: &DataHash, buffer: &mut [u8]) -> Result { + self.0.encode(&value.index, buffer)?; + self.encode(&value.nodes, buffer) } - fn decode(&mut self, buffer: &[u8]) -> DataHash { - let index: u64 = self.0.decode(buffer); - let nodes: Vec = self.decode(buffer); - DataHash { index, nodes } + fn decode(&mut self, buffer: &[u8]) -> Result { + let index: u64 = self.0.decode(buffer)?; + let nodes: Vec = self.decode(buffer)?; + Ok(DataHash { index, nodes }) } } impl CompactEncoding for HypercoreState { - fn preencode(&mut self, value: &DataSeek) { - self.0.preencode(&value.bytes); - self.preencode(&value.nodes); + fn preencode(&mut self, value: &DataSeek) -> Result { + self.0.preencode(&value.bytes)?; + self.preencode(&value.nodes) } - fn encode(&mut self, value: &DataSeek, buffer: &mut [u8]) { - self.0.encode(&value.bytes, buffer); - self.encode(&value.nodes, buffer); + fn encode(&mut self, value: &DataSeek, buffer: &mut [u8]) -> Result { + self.0.encode(&value.bytes, buffer)?; + self.encode(&value.nodes, buffer) } - fn decode(&mut self, buffer: &[u8]) -> DataSeek { - let bytes: u64 = self.0.decode(buffer); - let nodes: Vec = self.decode(buffer); - DataSeek { bytes, nodes } + fn decode(&mut self, buffer: &[u8]) -> Result { + let bytes: u64 = self.0.decode(buffer)?; + let nodes: Vec = self.decode(buffer)?; + Ok(DataSeek { bytes, nodes }) } } impl CompactEncoding for HypercoreState { - fn preencode(&mut self, value: &DataUpgrade) { - self.0.preencode(&value.start); - self.0.preencode(&value.length); - self.preencode(&value.nodes); - self.preencode(&value.additional_nodes); - self.0.preencode(&value.signature); - } - - fn encode(&mut self, value: &DataUpgrade, buffer: &mut [u8]) { - self.0.encode(&value.start, buffer); - self.0.encode(&value.length, buffer); - self.encode(&value.nodes, buffer); - self.encode(&value.additional_nodes, buffer); - self.0.encode(&value.signature, buffer); - } - - fn decode(&mut self, buffer: &[u8]) -> DataUpgrade { - let start: u64 = self.0.decode(buffer); - let length: u64 = self.0.decode(buffer); - let nodes: Vec = self.decode(buffer); - let additional_nodes: Vec = self.decode(buffer); - let signature: Vec = self.0.decode(buffer); - DataUpgrade { + fn preencode(&mut self, value: &DataUpgrade) -> Result { + self.0.preencode(&value.start)?; + self.0.preencode(&value.length)?; + self.preencode(&value.nodes)?; + self.preencode(&value.additional_nodes)?; + self.0.preencode(&value.signature) + } + + fn encode(&mut self, value: &DataUpgrade, buffer: &mut [u8]) -> Result { + self.0.encode(&value.start, buffer)?; + self.0.encode(&value.length, buffer)?; + self.encode(&value.nodes, buffer)?; + self.encode(&value.additional_nodes, buffer)?; + self.0.encode(&value.signature, buffer) + } + + fn decode(&mut self, buffer: &[u8]) -> Result { + let start: u64 = self.0.decode(buffer)?; + let length: u64 = self.0.decode(buffer)?; + let nodes: Vec = self.decode(buffer)?; + let additional_nodes: Vec = self.decode(buffer)?; + let signature: Vec = self.0.decode(buffer)?; + Ok(DataUpgrade { start, length, nodes, additional_nodes, signature, - } + }) } } diff --git a/src/oplog/entry.rs b/src/oplog/entry.rs index a74a7254..bdff2353 100644 --- a/src/oplog/entry.rs +++ b/src/oplog/entry.rs @@ -1,4 +1,4 @@ -use crate::encoding::{CompactEncoding, HypercoreState}; +use crate::encoding::{CompactEncoding, EncodingError, HypercoreState}; use crate::{common::BitfieldUpdate, Node}; /// Entry tree upgrade @@ -11,58 +11,65 @@ pub struct EntryTreeUpgrade { } impl CompactEncoding for HypercoreState { - fn preencode(&mut self, value: &EntryTreeUpgrade) { - self.0.preencode(&value.fork); - self.0.preencode(&value.ancestors); - self.0.preencode(&value.length); - self.0.preencode(&value.signature); + fn preencode(&mut self, value: &EntryTreeUpgrade) -> Result { + self.0.preencode(&value.fork)?; + self.0.preencode(&value.ancestors)?; + self.0.preencode(&value.length)?; + self.0.preencode(&value.signature) } - fn encode(&mut self, value: &EntryTreeUpgrade, buffer: &mut [u8]) { - self.0.encode(&value.fork, buffer); - self.0.encode(&value.ancestors, buffer); - self.0.encode(&value.length, buffer); - self.0.encode(&value.signature, buffer); + fn encode( + &mut self, + value: &EntryTreeUpgrade, + buffer: &mut [u8], + ) -> Result { + self.0.encode(&value.fork, buffer)?; + self.0.encode(&value.ancestors, buffer)?; + self.0.encode(&value.length, buffer)?; + self.0.encode(&value.signature, buffer) } - fn decode(&mut self, buffer: &[u8]) -> EntryTreeUpgrade { - let fork: u64 = self.0.decode(buffer); - let ancestors: u64 = self.0.decode(buffer); - let length: u64 = self.0.decode(buffer); - let signature: Box<[u8]> = self.0.decode(buffer); - EntryTreeUpgrade { + fn decode(&mut self, buffer: &[u8]) -> Result { + let fork: u64 = self.0.decode(buffer)?; + let ancestors: u64 = self.0.decode(buffer)?; + let length: u64 = self.0.decode(buffer)?; + let signature: Box<[u8]> = self.0.decode(buffer)?; + Ok(EntryTreeUpgrade { fork, ancestors, length, signature, - } + }) } } impl CompactEncoding for HypercoreState { - fn preencode(&mut self, value: &BitfieldUpdate) { - self.0.end += 1; - self.0.preencode(&value.start); - self.0.preencode(&value.length); + fn preencode(&mut self, value: &BitfieldUpdate) -> Result { + self.0.add_end(1)?; + self.0.preencode(&value.start)?; + self.0.preencode(&value.length) } - fn encode(&mut self, value: &BitfieldUpdate, buffer: &mut [u8]) { + fn encode( + &mut self, + value: &BitfieldUpdate, + buffer: &mut [u8], + ) -> Result { let flags: u8 = if value.drop { 1 } else { 0 }; - buffer[self.0.start] = flags; - self.0.start += 1; - self.0.encode(&value.start, buffer); - self.0.encode(&value.length, buffer); + self.0.set_byte_to_buffer(flags, buffer)?; + self.0.encode(&value.start, buffer)?; + self.0.encode(&value.length, buffer) } - fn decode(&mut self, buffer: &[u8]) -> BitfieldUpdate { - let flags = self.0.decode_u8(buffer); - let start: u64 = self.0.decode(buffer); - let length: u64 = self.0.decode(buffer); - BitfieldUpdate { + fn decode(&mut self, buffer: &[u8]) -> Result { + let flags = self.0.decode_u8(buffer)?; + let start: u64 = self.0.decode(buffer)?; + let length: u64 = self.0.decode(buffer)?; + Ok(BitfieldUpdate { drop: flags == 1, start, length, - } + }) } } @@ -77,79 +84,81 @@ pub struct Entry { } impl CompactEncoding for HypercoreState { - fn preencode(&mut self, value: &Entry) { - self.0.end += 1; // flags + fn preencode(&mut self, value: &Entry) -> Result { + self.0.add_end(1)?; // flags if value.user_data.len() > 0 { - self.0.preencode(&value.user_data); + self.0.preencode(&value.user_data)?; } if value.tree_nodes.len() > 0 { - self.preencode(&value.tree_nodes); + self.preencode(&value.tree_nodes)?; } if let Some(tree_upgrade) = &value.tree_upgrade { - self.preencode(tree_upgrade); + self.preencode(tree_upgrade)?; } if let Some(bitfield) = &value.bitfield { - self.preencode(bitfield); + self.preencode(bitfield)?; } + Ok(self.end()) } - fn encode(&mut self, value: &Entry, buffer: &mut [u8]) { - let start = self.0.start; - self.0.start += 1; + fn encode(&mut self, value: &Entry, buffer: &mut [u8]) -> Result { + let start = self.0.start(); + self.0.add_start(1)?; let mut flags: u8 = 0; if value.user_data.len() > 0 { flags = flags | 1; - self.0.encode(&value.user_data, buffer); + self.0.encode(&value.user_data, buffer)?; } if value.tree_nodes.len() > 0 { flags = flags | 2; - self.encode(&value.tree_nodes, buffer); + self.encode(&value.tree_nodes, buffer)?; } if let Some(tree_upgrade) = &value.tree_upgrade { flags = flags | 4; - self.encode(tree_upgrade, buffer); + self.encode(tree_upgrade, buffer)?; } if let Some(bitfield) = &value.bitfield { flags = flags | 8; - self.encode(bitfield, buffer); + self.encode(bitfield, buffer)?; } buffer[start] = flags; + Ok(self.0.start()) } - fn decode(&mut self, buffer: &[u8]) -> Entry { - let flags = self.0.decode_u8(buffer); + fn decode(&mut self, buffer: &[u8]) -> Result { + let flags = self.0.decode_u8(buffer)?; let user_data: Vec = if flags & 1 != 0 { - self.0.decode(buffer) + self.0.decode(buffer)? } else { vec![] }; let tree_nodes: Vec = if flags & 2 != 0 { - self.decode(buffer) + self.decode(buffer)? } else { vec![] }; let tree_upgrade: Option = if flags & 4 != 0 { - let value: EntryTreeUpgrade = self.decode(buffer); + let value: EntryTreeUpgrade = self.decode(buffer)?; Some(value) } else { None }; let bitfield: Option = if flags & 4 != 0 { - let value: BitfieldUpdate = self.decode(buffer); + let value: BitfieldUpdate = self.decode(buffer)?; Some(value) } else { None }; - Entry { + Ok(Entry { user_data, tree_nodes, tree_upgrade, bitfield, - } + }) } } diff --git a/src/oplog/header.rs b/src/oplog/header.rs index c2f02754..482ff8bb 100644 --- a/src/oplog/header.rs +++ b/src/oplog/header.rs @@ -1,4 +1,4 @@ -use compact_encoding::{CompactEncoding, State}; +use compact_encoding::{CompactEncoding, EncodingError, State}; use crate::crypto::{PublicKey, SecretKey}; use crate::PartialKeypair; @@ -64,27 +64,27 @@ impl HeaderTypes { } impl CompactEncoding for State { - fn preencode(&mut self, value: &HeaderTypes) { - self.preencode(&value.tree); - self.preencode(&value.bitfield); - self.preencode(&value.signer); + fn preencode(&mut self, value: &HeaderTypes) -> Result { + self.preencode(&value.tree)?; + self.preencode(&value.bitfield)?; + self.preencode(&value.signer) } - fn encode(&mut self, value: &HeaderTypes, buffer: &mut [u8]) { - self.encode(&value.tree, buffer); - self.encode(&value.bitfield, buffer); - self.encode(&value.signer, buffer); + fn encode(&mut self, value: &HeaderTypes, buffer: &mut [u8]) -> Result { + self.encode(&value.tree, buffer)?; + self.encode(&value.bitfield, buffer)?; + self.encode(&value.signer, buffer) } - fn decode(&mut self, buffer: &[u8]) -> HeaderTypes { - let tree: String = self.decode(buffer); - let bitfield: String = self.decode(buffer); - let signer: String = self.decode(buffer); - HeaderTypes { + fn decode(&mut self, buffer: &[u8]) -> Result { + let tree: String = self.decode(buffer)?; + let bitfield: String = self.decode(buffer)?; + let signer: String = self.decode(buffer)?; + Ok(HeaderTypes { tree, bitfield, signer, - } + }) } } @@ -109,81 +109,80 @@ impl HeaderTree { } impl CompactEncoding for State { - fn preencode(&mut self, value: &HeaderTree) { - self.preencode(&value.fork); - self.preencode(&value.length); - self.preencode(&value.root_hash); - self.preencode(&value.signature); + fn preencode(&mut self, value: &HeaderTree) -> Result { + self.preencode(&value.fork)?; + self.preencode(&value.length)?; + self.preencode(&value.root_hash)?; + self.preencode(&value.signature) } - fn encode(&mut self, value: &HeaderTree, buffer: &mut [u8]) { - self.encode(&value.fork, buffer); - self.encode(&value.length, buffer); - self.encode(&value.root_hash, buffer); - self.encode(&value.signature, buffer); + fn encode(&mut self, value: &HeaderTree, buffer: &mut [u8]) -> Result { + self.encode(&value.fork, buffer)?; + self.encode(&value.length, buffer)?; + self.encode(&value.root_hash, buffer)?; + self.encode(&value.signature, buffer) } - fn decode(&mut self, buffer: &[u8]) -> HeaderTree { - let fork: u64 = self.decode(buffer); - let length: u64 = self.decode(buffer); - let root_hash: Box<[u8]> = self.decode(buffer); - let signature: Box<[u8]> = self.decode(buffer); - HeaderTree { + fn decode(&mut self, buffer: &[u8]) -> Result { + let fork: u64 = self.decode(buffer)?; + let length: u64 = self.decode(buffer)?; + let root_hash: Box<[u8]> = self.decode(buffer)?; + let signature: Box<[u8]> = self.decode(buffer)?; + Ok(HeaderTree { fork, length, root_hash, signature, - } + }) } } /// NB: In Javascript's sodium the secret key contains in itself also the public key, so to /// maintain binary compatibility, we store the public key in the oplog now twice. impl CompactEncoding for State { - fn preencode(&mut self, value: &PartialKeypair) { - self.end += 1 + 32; + fn preencode(&mut self, value: &PartialKeypair) -> Result { + self.add_end(1 + 32)?; match &value.secret { Some(_) => { // Also add room for the public key - self.end += 1 + 64; - } - None => { - self.end += 1; + self.add_end(1 + 64) } + None => self.add_end(1), } } - fn encode(&mut self, value: &PartialKeypair, buffer: &mut [u8]) { + fn encode( + &mut self, + value: &PartialKeypair, + buffer: &mut [u8], + ) -> Result { let public_key_bytes: Box<[u8]> = value.public.as_bytes().to_vec().into_boxed_slice(); - self.encode(&public_key_bytes, buffer); + self.encode(&public_key_bytes, buffer)?; match &value.secret { Some(secret_key) => { let mut secret_key_bytes: Vec = Vec::with_capacity(64); secret_key_bytes.extend_from_slice(secret_key.as_bytes()); secret_key_bytes.extend_from_slice(&public_key_bytes); let secret_key_bytes: Box<[u8]> = secret_key_bytes.into_boxed_slice(); - self.encode(&secret_key_bytes, buffer); - } - None => { - buffer[self.start] = 0; - self.start += 1; + self.encode(&secret_key_bytes, buffer) } + None => self.set_byte_to_buffer(0, buffer), } } - fn decode(&mut self, buffer: &[u8]) -> PartialKeypair { - let public_key_bytes: Box<[u8]> = self.decode(buffer); - let secret_key_bytes: Box<[u8]> = self.decode(buffer); + fn decode(&mut self, buffer: &[u8]) -> Result { + let public_key_bytes: Box<[u8]> = self.decode(buffer)?; + let secret_key_bytes: Box<[u8]> = self.decode(buffer)?; let secret: Option = if secret_key_bytes.len() == 0 { None } else { Some(SecretKey::from_bytes(&secret_key_bytes[0..32]).unwrap()) }; - PartialKeypair { + Ok(PartialKeypair { public: PublicKey::from_bytes(&public_key_bytes).unwrap(), secret, - } + }) } } @@ -194,64 +193,62 @@ pub struct HeaderHints { } impl CompactEncoding for State { - fn preencode(&mut self, value: &HeaderHints) { - self.preencode(&value.reorgs); + fn preencode(&mut self, value: &HeaderHints) -> Result { + self.preencode(&value.reorgs) } - fn encode(&mut self, value: &HeaderHints, buffer: &mut [u8]) { - self.encode(&value.reorgs, buffer); + fn encode(&mut self, value: &HeaderHints, buffer: &mut [u8]) -> Result { + self.encode(&value.reorgs, buffer) } - fn decode(&mut self, buffer: &[u8]) -> HeaderHints { - HeaderHints { - reorgs: self.decode(buffer), - } + fn decode(&mut self, buffer: &[u8]) -> Result { + Ok(HeaderHints { + reorgs: self.decode(buffer)?, + }) } } impl CompactEncoding
for State { - fn preencode(&mut self, value: &Header) { - self.end += 1; // Version - self.preencode(&value.types); - self.preencode(&value.user_data); - self.preencode(&value.tree); - self.preencode(&value.signer); - self.preencode(&value.hints); - self.preencode(&value.contiguous_length); + fn preencode(&mut self, value: &Header) -> Result { + self.add_end(1)?; // Version + self.preencode(&value.types)?; + self.preencode(&value.user_data)?; + self.preencode(&value.tree)?; + self.preencode(&value.signer)?; + self.preencode(&value.hints)?; + self.preencode(&value.contiguous_length) } - fn encode(&mut self, value: &Header, buffer: &mut [u8]) { - buffer[self.start] = 0; // Version - self.start += 1; - self.encode(&value.types, buffer); - self.encode(&value.user_data, buffer); - self.encode(&value.tree, buffer); - self.encode(&value.signer, buffer); - self.encode(&value.hints, buffer); - self.encode(&value.contiguous_length, buffer); + fn encode(&mut self, value: &Header, buffer: &mut [u8]) -> Result { + self.set_byte_to_buffer(0, buffer)?; // Version + self.encode(&value.types, buffer)?; + self.encode(&value.user_data, buffer)?; + self.encode(&value.tree, buffer)?; + self.encode(&value.signer, buffer)?; + self.encode(&value.hints, buffer)?; + self.encode(&value.contiguous_length, buffer) } - fn decode(&mut self, buffer: &[u8]) -> Header { - let version: u8 = buffer[self.start]; - self.start += 1; + fn decode(&mut self, buffer: &[u8]) -> Result { + let version: u8 = self.decode_u8(buffer)?; if version != 0 { panic!("Unknown oplog version {}", version); } - let types: HeaderTypes = self.decode(buffer); - let user_data: Vec = self.decode(buffer); - let tree: HeaderTree = self.decode(buffer); - let signer: PartialKeypair = self.decode(buffer); - let hints: HeaderHints = self.decode(buffer); - let contiguous_length: u64 = self.decode(buffer); + let types: HeaderTypes = self.decode(buffer)?; + let user_data: Vec = self.decode(buffer)?; + let tree: HeaderTree = self.decode(buffer)?; + let signer: PartialKeypair = self.decode(buffer)?; + let hints: HeaderHints = self.decode(buffer)?; + let contiguous_length: u64 = self.decode(buffer)?; - Header { + Ok(Header { types, user_data, tree, signer, hints, contiguous_length, - } + }) } } @@ -262,58 +259,61 @@ mod tests { use crate::crypto::generate_keypair; #[test] - fn encode_header_types() { + fn encode_header_types() -> Result<(), EncodingError> { let mut enc_state = State::new_with_start_and_end(8, 8); let header_types = HeaderTypes::new(); - enc_state.preencode(&header_types); + enc_state.preencode(&header_types)?; let mut buffer = enc_state.create_buffer(); - enc_state.encode(&header_types, &mut buffer); + enc_state.encode(&header_types, &mut buffer)?; let mut dec_state = State::from_buffer(&buffer); - dec_state.start = 8; - let header_types_ret: HeaderTypes = dec_state.decode(&buffer); + dec_state.add_start(8)?; + let header_types_ret: HeaderTypes = dec_state.decode(&buffer)?; assert_eq!(header_types, header_types_ret); + Ok(()) } #[test] - fn encode_partial_key_pair() { + fn encode_partial_key_pair() -> Result<(), EncodingError> { let mut enc_state = State::new(); let key_pair = generate_keypair(); let key_pair = PartialKeypair { public: key_pair.public, secret: Some(key_pair.secret), }; - enc_state.preencode(&key_pair); + enc_state.preencode(&key_pair)?; let mut buffer = enc_state.create_buffer(); // Pub key: 1 byte for length, 32 bytes for content // Sec key: 1 byte for length, 64 bytes for data let expected_len = 1 + 32 + 1 + 64; assert_eq!(buffer.len(), expected_len); - assert_eq!(enc_state.end, expected_len); - assert_eq!(enc_state.start, 0); - enc_state.encode(&key_pair, &mut buffer); + assert_eq!(enc_state.end(), expected_len); + assert_eq!(enc_state.start(), 0); + enc_state.encode(&key_pair, &mut buffer)?; let mut dec_state = State::from_buffer(&buffer); - let key_pair_ret: PartialKeypair = dec_state.decode(&buffer); + let key_pair_ret: PartialKeypair = dec_state.decode(&buffer)?; assert_eq!(key_pair.public, key_pair_ret.public); assert_eq!( key_pair.secret.unwrap().as_bytes(), key_pair_ret.secret.unwrap().as_bytes() ); + Ok(()) } #[test] - fn encode_tree() { + fn encode_tree() -> Result<(), EncodingError> { let mut enc_state = State::new(); let tree = HeaderTree::new(); - enc_state.preencode(&tree); + enc_state.preencode(&tree)?; let mut buffer = enc_state.create_buffer(); - enc_state.encode(&tree, &mut buffer); + enc_state.encode(&tree, &mut buffer)?; let mut dec_state = State::from_buffer(&buffer); - let tree_ret: HeaderTree = dec_state.decode(&buffer); + let tree_ret: HeaderTree = dec_state.decode(&buffer)?; assert_eq!(tree, tree_ret); + Ok(()) } #[test] - fn encode_header() { + fn encode_header() -> Result<(), EncodingError> { let mut enc_state = State::new(); let key_pair = generate_keypair(); let key_pair = PartialKeypair { @@ -321,14 +321,15 @@ mod tests { secret: Some(key_pair.secret), }; let header = Header::new(key_pair); - enc_state.preencode(&header); + enc_state.preencode(&header)?; let mut buffer = enc_state.create_buffer(); - enc_state.encode(&header, &mut buffer); + enc_state.encode(&header, &mut buffer)?; let mut dec_state = State::from_buffer(&buffer); - let header_ret: Header = dec_state.decode(&buffer); + let header_ret: Header = dec_state.decode(&buffer)?; assert_eq!(header.signer.public, header_ret.signer.public); assert_eq!(header.tree.fork, header_ret.tree.fork); assert_eq!(header.tree.length, header_ret.tree.length); assert_eq!(header.types, header_ret.types); + Ok(()) } } diff --git a/src/oplog/mod.rs b/src/oplog/mod.rs index 1b77393e..6743a57f 100644 --- a/src/oplog/mod.rs +++ b/src/oplog/mod.rs @@ -105,14 +105,14 @@ impl Oplog { if let Some(mut h2_outcome) = h2_outcome { let header_bits = [h1_outcome.header_bit, h2_outcome.header_bit]; let header: Header = if header_bits[0] == header_bits[1] { - (*h1_outcome.state).decode(&existing) + (*h1_outcome.state).decode(&existing)? } else { - (*h2_outcome.state).decode(&existing) + (*h2_outcome.state).decode(&existing)? }; (header, header_bits) } else { ( - (*h1_outcome.state).decode(&existing), + (*h1_outcome.state).decode(&existing)?, [h1_outcome.header_bit, h1_outcome.header_bit], ) }; @@ -133,12 +133,12 @@ impl Oplog { }; OplogOpenOutcome::new( oplog, - (*h2_outcome.state).decode(&existing), + (*h2_outcome.state).decode(&existing)?, Box::new([]), ) } else if let Some(key_pair) = key_pair { // There is nothing in the oplog, start from new given key pair. - Self::new(key_pair.clone()) + Self::new(key_pair.clone())? } else { // The storage is empty and no key pair given, erroring return Err(HypercoreError::EmptyStorage { @@ -155,10 +155,10 @@ impl Oplog { if let Some(mut entry_outcome) = Self::validate_leader(entry_offset as usize, &existing)? { - let entry: Entry = entry_outcome.state.decode(&existing); + let entry: Entry = entry_outcome.state.decode(&existing)?; entries.push(entry); partials.push(entry_outcome.partial_bit); - entry_offset = (*entry_outcome.state).end; + entry_offset = (*entry_outcome.state).end(); } else { break; } @@ -182,7 +182,7 @@ impl Oplog { bitfield_update: Option, atomic: bool, header: &Header, - ) -> OplogCreateHeaderOutcome { + ) -> Result { let tree_nodes: Vec = changeset.nodes.clone(); let mut header: Header = header.clone(); let entry: Entry = if changeset.upgraded { @@ -218,14 +218,14 @@ impl Oplog { } }; - OplogCreateHeaderOutcome { + Ok(OplogCreateHeaderOutcome { header, - infos_to_flush: self.append_entries(&[entry], atomic), - } + infos_to_flush: self.append_entries(&[entry], atomic)?, + }) } /// Clears a segment, returns infos to write to storage. - pub fn clear(&mut self, start: u64, end: u64) -> Box<[StoreInfo]> { + pub fn clear(&mut self, start: u64, end: u64) -> Result, HypercoreError> { let entry: Entry = Entry { user_data: vec![], tree_nodes: vec![], @@ -240,73 +240,77 @@ impl Oplog { } /// Flushes pending changes, returns infos to write to storage. - pub fn flush(&mut self, header: &Header) -> Box<[StoreInfo]> { - let (new_header_bits, infos_to_flush) = Self::insert_header(header, 0, self.header_bits); + pub fn flush(&mut self, header: &Header) -> Result, HypercoreError> { + let (new_header_bits, infos_to_flush) = Self::insert_header(header, 0, self.header_bits)?; self.entries_byte_length = 0; self.entries_length = 0; self.header_bits = new_header_bits; - infos_to_flush + Ok(infos_to_flush) } /// Appends a batch of entries to the Oplog. - fn append_entries(&mut self, batch: &[Entry], atomic: bool) -> Box<[StoreInfo]> { + fn append_entries( + &mut self, + batch: &[Entry], + atomic: bool, + ) -> Result, HypercoreError> { let len = batch.len(); let header_bit = self.get_current_header_bit(); // Leave room for leaders let mut state = HypercoreState::new_with_start_and_end(0, len * 8); for entry in batch.iter() { - state.preencode(entry); + state.preencode(entry)?; } let mut buffer = state.create_buffer(); for i in 0..len { let entry = &batch[i]; - (*state).start += 8; - let start = state.start; + (*state).add_start(8)?; + let start = state.start(); let partial_bit: bool = atomic && i < len - 1; - state.encode(entry, &mut buffer); + state.encode(entry, &mut buffer)?; Self::prepend_leader( - state.start - start, + state.start() - start, header_bit, partial_bit, &mut state, &mut buffer, - ); + )?; } let index = OplogSlot::Entries as u64 + self.entries_byte_length; self.entries_length += len as u64; self.entries_byte_length += buffer.len() as u64; - vec![StoreInfo::new_content(Store::Oplog, index, &buffer)].into_boxed_slice() + Ok(vec![StoreInfo::new_content(Store::Oplog, index, &buffer)].into_boxed_slice()) } - fn new(key_pair: PartialKeypair) -> OplogOpenOutcome { + fn new(key_pair: PartialKeypair) -> Result { let entries_length: u64 = 0; let entries_byte_length: u64 = 0; let header = Header::new(key_pair); let (header_bits, infos_to_flush) = - Self::insert_header(&header, entries_byte_length, INITIAL_HEADER_BITS); + Self::insert_header(&header, entries_byte_length, INITIAL_HEADER_BITS)?; let oplog = Oplog { header_bits, entries_length, entries_byte_length, }; - OplogOpenOutcome::from_create_header_outcome( + Ok(OplogOpenOutcome::from_create_header_outcome( oplog, OplogCreateHeaderOutcome { header, infos_to_flush, }, - ) + )) } fn insert_header( header: &Header, entries_byte_length: u64, current_header_bits: [bool; 2], - ) -> ([bool; 2], Box<[StoreInfo]>) { + ) -> Result<([bool; 2], Box<[StoreInfo]>), HypercoreError> { // The first 8 bytes will be filled with `prepend_leader`. let data_start_index: usize = 8; let mut state = HypercoreState::new_with_start_and_end(data_start_index, data_start_index); @@ -324,34 +328,34 @@ impl Oplog { } // Preencode the new header - (*state).preencode(header); + (*state).preencode(header)?; // Create a buffer for the needed data let mut buffer = state.create_buffer(); // Encode the header - (*state).encode(header, &mut buffer); + (*state).encode(header, &mut buffer)?; // Finally prepend the buffer's 8 first bytes with a CRC, len and right bits Self::prepend_leader( - state.end - data_start_index, + state.end() - data_start_index, header_bit, false, &mut state, &mut buffer, - ); + )?; // The oplog is always truncated to the minimum byte size, which is right after // the all of the entries in the oplog finish. let truncate_index = OplogSlot::Entries as u64 + entries_byte_length; - ( + Ok(( new_header_bits, vec![ StoreInfo::new_content(Store::Oplog, oplog_slot as u64, &buffer), StoreInfo::new_truncate(Store::Oplog, truncate_index), ] .into_boxed_slice(), - ) + )) } /// Prepends given `State` with 4 bytes of CRC followed by 4 bytes containing length of @@ -365,21 +369,24 @@ impl Oplog { partial_bit: bool, state: &mut HypercoreState, buffer: &mut Box<[u8]>, - ) { + ) -> Result<(), HypercoreError> { // The 4 bytes right before start of data is the length in 8+8+8+6=30 bits. The 31st bit is // the partial bit and 32nd bit the header bit. - (*state).start = (*state).start - len - 4; + let start = (*state).start(); + (*state).set_start(start - len - 4)?; let len_u32: u32 = len.try_into().unwrap(); let partial_bit: u32 = if partial_bit { 2 } else { 0 }; let header_bit: u32 = if header_bit { 1 } else { 0 }; let combined: u32 = (len_u32 << 2) | header_bit | partial_bit; - state.encode_u32(combined, buffer); + state.encode_u32(combined, buffer)?; // Before that, is a 4 byte CRC32 that is a checksum of the above encoded 4 bytes and the // content. - state.start = state.start - 8; - let checksum = crc32fast::hash(&buffer[state.start + 4..state.start + 8 + len]); - state.encode_u32(checksum, buffer); + let start = state.start(); + state.set_start(start - 8)?; + let checksum = crc32fast::hash(&buffer[state.start() + 4..state.start() + 8 + len]); + state.encode_u32(checksum, buffer)?; + Ok(()) } /// Validates that leader at given index is valid, and returns header and partial bits and @@ -392,23 +399,25 @@ impl Oplog { return Ok(None); } let mut state = HypercoreState::new_with_start_and_end(index, buffer.len()); - let stored_checksum: u32 = state.decode_u32(buffer); - let combined: u32 = state.decode_u32(buffer); + let stored_checksum: u32 = state.decode_u32(buffer)?; + let combined: u32 = state.decode_u32(buffer)?; let len = usize::try_from(combined >> 2) .expect("Attempted converting to a 32 bit usize on below 32 bit system"); // NB: In the Javascript version IIUC zero length is caught only with a mismatch // of checksums, which is silently interpreted to only mean "no value". That doesn't sound good: // better to throw an error on mismatch and let the caller at least log the problem. - if len == 0 || state.end - state.start < len { + if len == 0 || state.end() - state.start() < len { return Ok(None); } let header_bit = combined & 1 == 1; let partial_bit = combined & 2 == 2; - state.start = index + 8; - state.end = state.start + len; - let calculated_checksum = crc32fast::hash(&buffer[index + 4..state.end]); + let new_start = index + 8; + state.set_end(new_start + len); + state.set_start(new_start)?; + + let calculated_checksum = crc32fast::hash(&buffer[index + 4..state.end()]); if calculated_checksum != stored_checksum { return Err(HypercoreError::InvalidChecksum { context: "Calculated signature does not match oplog signature".to_string(), diff --git a/src/tree/merkle_tree.rs b/src/tree/merkle_tree.rs index 3e9fca81..716da2f8 100644 --- a/src/tree/merkle_tree.rs +++ b/src/tree/merkle_tree.rs @@ -82,7 +82,7 @@ impl MerkleTree { }); } let data = infos[i].data.as_ref().unwrap(); - let node = node_from_bytes(&index, data); + let node = node_from_bytes(&index, data)?; byte_length += node.length; // This is totalSpan in Javascript length += 2 * ((node.index - length) + 1); @@ -172,7 +172,7 @@ impl MerkleTree { ) -> Result, NodeByteRange>, HypercoreError> { let index = self.validate_hypercore_index(hypercore_index)?; // Get nodes out of incoming infos - let nodes: IntMap> = self.infos_to_nodes(infos); + let nodes: IntMap> = self.infos_to_nodes(infos)?; // Start with getting the requested node, which will get the length // of the byte range @@ -285,7 +285,7 @@ impl MerkleTree { let head = length * 2; let mut full_roots = vec![]; flat_tree::full_roots(head, &mut full_roots); - let nodes: IntMap> = self.infos_to_nodes(infos); + let nodes: IntMap> = self.infos_to_nodes(infos)?; let mut changeset = self.changeset(); let mut instructions: Vec = Vec::new(); @@ -339,7 +339,7 @@ impl MerkleTree { upgrade: Option<&RequestUpgrade>, infos: Option<&[StoreInfo]>, ) -> Result, ValuelessProof>, HypercoreError> { - let nodes: IntMap> = self.infos_to_nodes(infos); + let nodes: IntMap> = self.infos_to_nodes(infos)?; let mut instructions: Vec = Vec::new(); let fork = self.fork; let signature = self.signature; @@ -518,7 +518,7 @@ impl MerkleTree { public_key: &PublicKey, infos: Option<&[StoreInfo]>, ) -> Result, MerkleTreeChangeset>, HypercoreError> { - let nodes: IntMap> = self.infos_to_nodes(infos); + let nodes: IntMap> = self.infos_to_nodes(infos)?; let mut instructions: Vec = Vec::new(); let mut changeset = self.changeset(); @@ -583,7 +583,7 @@ impl MerkleTree { return Ok(Either::Right(0)); } - let nodes: IntMap> = self.infos_to_nodes(infos); + let nodes: IntMap> = self.infos_to_nodes(infos)?; let mut count: u64 = 0; while !iter.contains(head) { match self.optional_node(iter.index(), &nodes)? { @@ -665,8 +665,12 @@ impl MerkleTree { let mut infos_to_flush: Vec = Vec::with_capacity(self.unflushed.len()); for (_, node) in self.unflushed.drain() { let (mut state, mut buffer) = State::new_with_size(40); - state.encode_u64(node.length, &mut buffer); - state.encode_fixed_32(&node.hash, &mut buffer); + state + .encode_u64(node.length, &mut buffer) + .expect("Encoding u64 should not fail"); + state + .encode_fixed_32(&node.hash, &mut buffer) + .expect("Encoding fixed 32 bytes should not fail"); infos_to_flush.push(StoreInfo::new_content( Store::Tree, node.index * 40, @@ -702,7 +706,7 @@ impl MerkleTree { infos: Option<&[StoreInfo]>, ) -> Result, u64>, HypercoreError> { // Get nodes out of incoming infos - let nodes: IntMap> = self.infos_to_nodes(infos); + let nodes: IntMap> = self.infos_to_nodes(infos)?; // Get offset let offset_result = self.byte_offset_from_nodes(index, &nodes)?; // Get offset @@ -1302,14 +1306,17 @@ impl MerkleTree { } } - fn infos_to_nodes(&mut self, infos: Option<&[StoreInfo]>) -> IntMap> { + fn infos_to_nodes( + &mut self, + infos: Option<&[StoreInfo]>, + ) -> Result>, HypercoreError> { match infos { Some(infos) => { let mut nodes: IntMap> = IntMap::with_capacity(infos.len()); for info in infos { let index = index_from_info(&info); if !info.miss { - let node = node_from_bytes(&index, info.data.as_ref().unwrap()); + let node = node_from_bytes(&index, info.data.as_ref().unwrap())?; #[cfg(feature = "cache")] if !node.blank { if let Some(node_cache) = &self.node_cache { @@ -1321,9 +1328,9 @@ impl MerkleTree { nodes.insert(index, None); } } - nodes + Ok(nodes) } - None => IntMap::new(), + None => Ok(IntMap::new()), } } } @@ -1474,12 +1481,12 @@ fn index_from_info(info: &StoreInfo) -> u64 { info.index / NODE_SIZE } -fn node_from_bytes(index: &u64, data: &[u8]) -> Node { +fn node_from_bytes(index: &u64, data: &[u8]) -> Result { let len_buf = &data[..8]; let hash = &data[8..]; let mut state = State::from_buffer(len_buf); - let len = state.decode_u64(len_buf); - Node::new(*index, hash.to_vec(), len) + let len = state.decode_u64(len_buf)?; + Ok(Node::new(*index, hash.to_vec(), len)) } #[derive(Debug, Copy, Clone)] From ec9e329411896bfff21770448b6dc1d7e3958926 Mon Sep 17 00:00:00 2001 From: Timo Tiuraniemi Date: Tue, 4 Apr 2023 09:20:01 +0300 Subject: [PATCH 116/157] Export also EncodingErrorKind --- src/encoding.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/encoding.rs b/src/encoding.rs index b6f1ce3a..6960ad9a 100644 --- a/src/encoding.rs +++ b/src/encoding.rs @@ -1,5 +1,5 @@ //! Hypercore-specific compact encodings -pub use compact_encoding::{CompactEncoding, EncodingError, State}; +pub use compact_encoding::{CompactEncoding, EncodingError, EncodingErrorKind, State}; use std::ops::{Deref, DerefMut}; use crate::{ From 91938c1b3c62d6d4fe31bc03fbc51e350345bc82 Mon Sep 17 00:00:00 2001 From: Timo Tiuraniemi Date: Tue, 4 Apr 2023 14:27:00 +0300 Subject: [PATCH 117/157] Use merged compact encoding --- Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index 89477c32..9d812b7e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -27,7 +27,7 @@ byteorder = "1.3.4" ed25519-dalek = "1.0.1" thiserror = "1" tracing = "0.1" -compact-encoding = { git = "https://github.com/datrs/compact-encoding", branch = "error" } +compact-encoding = { git = "https://github.com/datrs/compact-encoding", branch = "main" } flat-tree = { git = "https://github.com/ttiurani/flat-tree", branch = "v10" } lazy_static = "1.4.0" memory-pager = "0.9.0" From abd89862318ee81d4196f78e20a978df5d5b1132 Mon Sep 17 00:00:00 2001 From: Timo Tiuraniemi Date: Tue, 4 Apr 2023 14:38:33 +0300 Subject: [PATCH 118/157] Bump version --- Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index 9d812b7e..b938e304 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "hypercore" -version = "0.12.0-alpha.0" +version = "0.12.0-alpha.1" license = "MIT OR Apache-2.0" description = "Secure, distributed, append-only log" documentation = "https://docs.rs/hypercore" From 7e38e0564a7817a27211213de4d832eecfb52e0a Mon Sep 17 00:00:00 2001 From: Timo Tiuraniemi Date: Wed, 5 Apr 2023 10:28:38 +0300 Subject: [PATCH 119/157] Remove v9 merkle.rs --- src/crypto/merkle.rs | 71 -------------------------------------------- src/crypto/mod.rs | 2 -- 2 files changed, 73 deletions(-) delete mode 100644 src/crypto/merkle.rs diff --git a/src/crypto/merkle.rs b/src/crypto/merkle.rs deleted file mode 100644 index bbd1cb22..00000000 --- a/src/crypto/merkle.rs +++ /dev/null @@ -1,71 +0,0 @@ -use crate::common::Node; -use crate::crypto::Hash; -use merkle_tree_stream::{HashMethods, MerkleTreeStream, NodeKind, PartialNode}; -use std::sync::Arc; - -#[derive(Debug)] -struct H; - -impl HashMethods for H { - type Node = Node; - type Hash = Hash; - - fn leaf(&self, leaf: &PartialNode, _roots: &[Arc]) -> Self::Hash { - match leaf.data() { - NodeKind::Leaf(data) => Hash::from_leaf(&data), - NodeKind::Parent => unreachable!(), - } - } - - fn parent(&self, left: &Self::Node, right: &Self::Node) -> Self::Hash { - Hash::from_hashes(left, right) - } -} - -/// Merkle Tree Stream -#[derive(Debug)] -pub struct Merkle { - stream: MerkleTreeStream, - nodes: Vec>, -} - -impl Default for Merkle { - fn default() -> Self { - Merkle::new() - } -} - -impl Merkle { - /// Create a new instance. - // TODO: figure out the right allocation size for `roots` and `nodes`. - pub fn new() -> Self { - Self { - nodes: vec![], - stream: MerkleTreeStream::new(H, vec![]), - } - } - - pub fn from_nodes(nodes: Vec) -> Self { - let nodes = nodes.into_iter().map(Arc::new).collect::>(); - Self { - stream: MerkleTreeStream::new(H, nodes.clone()), - nodes, - } - } - - /// Access the next item. - // TODO: remove extra conversion alloc. - pub fn next(&mut self, data: &[u8]) { - self.stream.next(&data, &mut self.nodes); - } - - /// Get the roots vector. - pub fn roots(&self) -> &Vec> { - self.stream.roots() - } - - /// Get the nodes from the struct. - pub fn nodes(&self) -> &Vec> { - &self.nodes - } -} diff --git a/src/crypto/mod.rs b/src/crypto/mod.rs index 1909d2ea..699759bf 100644 --- a/src/crypto/mod.rs +++ b/src/crypto/mod.rs @@ -2,11 +2,9 @@ mod hash; mod key_pair; -mod merkle; pub use self::hash::signable_tree; pub use self::hash::Hash; pub use self::key_pair::{ generate as generate_keypair, sign, verify, PublicKey, SecretKey, Signature, }; -pub use self::merkle::Merkle; From 46e91decafb7e4e10bb96a3b304399a3aceb1376 Mon Sep 17 00:00:00 2001 From: Timo Tiuraniemi Date: Wed, 5 Apr 2023 10:29:04 +0300 Subject: [PATCH 120/157] Fix build warnings --- src/bitfield/dynamic.rs | 7 ++++--- src/bitfield/fixed.rs | 11 ++++------- src/tree/merkle_tree.rs | 2 -- 3 files changed, 8 insertions(+), 12 deletions(-) diff --git a/src/bitfield/dynamic.rs b/src/bitfield/dynamic.rs index 394a80ed..87ab9c0b 100644 --- a/src/bitfield/dynamic.rs +++ b/src/bitfield/dynamic.rs @@ -44,7 +44,7 @@ impl DynamicBitfield { let parent_index: u64 = (data_index / FIXED_BITFIELD_LENGTH) as u64; pages.insert( parent_index, - RefCell::new(FixedBitfield::from_data(parent_index, data_index, &data)), + RefCell::new(FixedBitfield::from_data(data_index, &data)), ); if parent_index > biggest_page_index { biggest_page_index = parent_index; @@ -96,13 +96,14 @@ impl DynamicBitfield { } } + #[allow(dead_code)] pub fn set(&mut self, index: u64, value: bool) -> bool { let j = index & (DYNAMIC_BITFIELD_PAGE_SIZE as u64 - 1); let i = (index - j) / DYNAMIC_BITFIELD_PAGE_SIZE as u64; if !self.pages.contains_key(i) { if value { - self.pages.insert(i, RefCell::new(FixedBitfield::new(i))); + self.pages.insert(i, RefCell::new(FixedBitfield::new())); if i > self.biggest_page_index { self.biggest_page_index = i; } @@ -137,7 +138,7 @@ impl DynamicBitfield { while length > 0 { if !self.pages.contains_key(i) { - self.pages.insert(i, RefCell::new(FixedBitfield::new(i))); + self.pages.insert(i, RefCell::new(FixedBitfield::new())); if i > self.biggest_page_index { self.biggest_page_index = i; } diff --git a/src/bitfield/fixed.rs b/src/bitfield/fixed.rs index 3492d080..0e607e81 100644 --- a/src/bitfield/fixed.rs +++ b/src/bitfield/fixed.rs @@ -14,21 +14,19 @@ use std::convert::TryInto; /// https://github.com/hypercore-protocol/hypercore/commit/6392021b11d53041a446e9021c7d79350a052d3d #[derive(Debug)] pub struct FixedBitfield { - pub(crate) parent_index: u64, pub(crate) dirty: bool, bitfield: [u32; FIXED_BITFIELD_LENGTH], } impl FixedBitfield { - pub fn new(parent_index: u64) -> Self { + pub fn new() -> Self { Self { - parent_index, dirty: false, bitfield: [0; FIXED_BITFIELD_LENGTH], } } - pub fn from_data(parent_index: u64, data_index: usize, data: &[u8]) -> Self { + pub fn from_data(data_index: usize, data: &[u8]) -> Self { let mut bitfield = [0; FIXED_BITFIELD_LENGTH]; if data.len() >= data_index + 4 { let mut i = data_index; @@ -43,7 +41,6 @@ impl FixedBitfield { } } Self { - parent_index, dirty: false, bitfield, } @@ -171,7 +168,7 @@ mod tests { #[test] fn bitfield_fixed_get_and_set() { - let mut bitfield = FixedBitfield::new(0); + let mut bitfield = FixedBitfield::new(); assert_value_range(&bitfield, 0, 9, false); assert_eq!(bitfield.index_of(true, 0), None); assert_eq!(bitfield.index_of(false, 0), Some(0)); @@ -210,7 +207,7 @@ mod tests { #[test] fn bitfield_fixed_set_range() { - let mut bitfield = FixedBitfield::new(0); + let mut bitfield = FixedBitfield::new(); bitfield.set_range(0, 2, true); assert_value_range(&bitfield, 0, 2, true); assert_value_range(&bitfield, 3, 61, false); diff --git a/src/tree/merkle_tree.rs b/src/tree/merkle_tree.rs index 716da2f8..6f0bf260 100644 --- a/src/tree/merkle_tree.rs +++ b/src/tree/merkle_tree.rs @@ -360,7 +360,6 @@ impl MerkleTree { let mut sub_tree = head; let mut p = LocalProof { - indexed: None, seek: None, nodes: None, upgrade: None, @@ -1548,7 +1547,6 @@ fn normalize_data(block: Option<&DataBlock>, hash: Option<&DataHash>) -> Option< /// Struct to use for local building of proof #[derive(Debug, Clone)] struct LocalProof { - pub indexed: Option, pub seek: Option>, pub nodes: Option>, pub upgrade: Option>, From 425fc3d99b6d48bf7b44340136b47325053ed4af Mon Sep 17 00:00:00 2001 From: Timo Tiuraniemi Date: Wed, 5 Apr 2023 11:08:40 +0300 Subject: [PATCH 121/157] Fix some clippy warnings --- src/bitfield/dynamic.rs | 32 +++++++------------- src/bitfield/fixed.rs | 30 ++++++++----------- src/common/node.rs | 6 ++-- src/common/peer.rs | 2 +- src/core.rs | 36 ++++++++++------------- src/crypto/hash.rs | 18 ++++++------ src/data/mod.rs | 2 +- src/encoding.rs | 9 +++++- src/oplog/entry.rs | 16 +++++----- src/storage/mod.rs | 8 ++--- src/tree/merkle_tree.rs | 49 ++++++++++++------------------- src/tree/merkle_tree_changeset.rs | 6 ++-- 12 files changed, 95 insertions(+), 119 deletions(-) diff --git a/src/bitfield/dynamic.rs b/src/bitfield/dynamic.rs index 87ab9c0b..f5c68894 100644 --- a/src/bitfield/dynamic.rs +++ b/src/bitfield/dynamic.rs @@ -184,12 +184,7 @@ impl DynamicBitfield { // It wasn't found on the first page, now get the keys that are bigger // than the given index and sort them. - let mut keys: Vec<&u64> = self - .pages - .keys() - .into_iter() - .filter(|key| **key > first_page) - .collect(); + let mut keys: Vec<&u64> = self.pages.keys().filter(|key| **key > first_page).collect(); keys.sort(); for key in keys { if let Some(p) = self.pages.get(*key) { @@ -236,12 +231,7 @@ impl DynamicBitfield { // It wasn't found on the last page, now get the keys that are smaller // than the given index and sort them. - let mut keys: Vec<&u64> = self - .pages - .keys() - .into_iter() - .filter(|key| **key < last_page) - .collect(); + let mut keys: Vec<&u64> = self.pages.keys().filter(|key| **key < last_page).collect(); keys.sort(); keys.reverse(); @@ -260,7 +250,7 @@ impl DynamicBitfield { // a missing page. let mut i = last_page; let mut j = last_index as u32; - while i == last_page || i <= 0 { + while i == last_page || i == 0 { if let Some(p) = self.pages.get(i) { if let Some(index) = p.borrow().last_index_of(value, j) { return Some(i * DYNAMIC_BITFIELD_PAGE_SIZE as u64 + index as u64); @@ -308,7 +298,7 @@ mod tests { assert_eq!(bitfield.last_index_of(false, 10000000), Some(10000000)); bitfield.set(0, true); - assert_eq!(bitfield.get(0), true); + assert!(bitfield.get(0)); assert_eq!(bitfield.index_of(true, 0), Some(0)); assert_eq!(bitfield.index_of(false, 0), Some(1)); assert_eq!(bitfield.last_index_of(true, 9), Some(0)); @@ -318,18 +308,18 @@ mod tests { assert_value_range(&bitfield, 1, 63, false); bitfield.set(31, true); - assert_eq!(bitfield.get(31), true); + assert!(bitfield.get(31)); assert_value_range(&bitfield, 32, 32, false); - assert_eq!(bitfield.get(32), false); + assert!(!bitfield.get(32)); bitfield.set(32, true); - assert_eq!(bitfield.get(32), true); + assert!(bitfield.get(32)); assert_value_range(&bitfield, 33, 31, false); assert_value_range(&bitfield, 32760, 8, false); - assert_eq!(bitfield.get(32767), false); + assert!(!bitfield.get(32767)); bitfield.set(32767, true); - assert_eq!(bitfield.get(32767), true); + assert!(bitfield.get(32767)); assert_value_range(&bitfield, 32760, 7, false); // Now for over one fixed bitfield values @@ -338,7 +328,7 @@ mod tests { assert_value_range(&bitfield, 32769, 9, false); bitfield.set(10000000, true); - assert_eq!(bitfield.get(10000000), true); + assert!(bitfield.get(10000000)); assert_value_range(&bitfield, 9999990, 10, false); assert_value_range(&bitfield, 10000001, 9, false); assert_eq!(bitfield.index_of(false, 32767), Some(32769)); @@ -358,7 +348,7 @@ mod tests { assert_value_range(&bitfield, 5, 59, false); bitfield.set_range(1, 3, false); - assert_eq!(bitfield.get(0), true); + assert!(bitfield.get(0)); assert_value_range(&bitfield, 1, 3, false); assert_value_range(&bitfield, 4, 1, true); assert_value_range(&bitfield, 5, 59, false); diff --git a/src/bitfield/fixed.rs b/src/bitfield/fixed.rs index 0e607e81..874c18c8 100644 --- a/src/bitfield/fixed.rs +++ b/src/bitfield/fixed.rs @@ -32,7 +32,7 @@ impl FixedBitfield { let mut i = data_index; let limit = std::cmp::min(data_index + FIXED_BITFIELD_BYTES_LENGTH, data.len()) - 4; while i <= limit { - let value: u32 = ((data[i] as u32) << 0) + let value: u32 = (data[i] as u32) | ((data[i + 1] as u32) << 8) | ((data[i + 2] as u32) << 16) | ((data[i + 3] as u32) << 24); @@ -81,10 +81,8 @@ impl FixedBitfield { if (self.bitfield[i] & mask) != 0 { return false; } - } else { - if (self.bitfield[i] & mask) == 0 { - return false; - } + } else if (self.bitfield[i] & mask) == 0 { + return false; } self.bitfield[i] ^= mask; true @@ -120,11 +118,9 @@ impl FixedBitfield { self.bitfield[i] |= mask; changed = true; } - } else { - if (self.bitfield[i] & mask) != 0 { - self.bitfield[i] &= !mask; - changed = true; - } + } else if (self.bitfield[i] & mask) != 0 { + self.bitfield[i] &= !mask; + changed = true; } remaining -= (n - offset) as i64; @@ -176,7 +172,7 @@ mod tests { assert_eq!(bitfield.last_index_of(false, 9), Some(9)); bitfield.set(0, true); - assert_eq!(bitfield.get(0), true); + assert!(bitfield.get(0)); assert_eq!(bitfield.index_of(true, 0), Some(0)); assert_eq!(bitfield.index_of(false, 0), Some(1)); assert_eq!(bitfield.last_index_of(true, 9), Some(0)); @@ -185,20 +181,20 @@ mod tests { assert_value_range(&bitfield, 1, 63, false); bitfield.set(31, true); - assert_eq!(bitfield.get(31), true); + assert!(bitfield.get(31)); assert_eq!(bitfield.index_of(true, 1), Some(31)); assert_eq!(bitfield.index_of(false, 31), Some(32)); assert_value_range(&bitfield, 32, 32, false); - assert_eq!(bitfield.get(32), false); + assert!(!bitfield.get(32)); bitfield.set(32, true); - assert_eq!(bitfield.get(32), true); + assert!(bitfield.get(32)); assert_value_range(&bitfield, 33, 31, false); assert_value_range(&bitfield, 32760, 8, false); - assert_eq!(bitfield.get(32767), false); + assert!(!bitfield.get(32767)); bitfield.set(32767, true); - assert_eq!(bitfield.get(32767), true); + assert!(bitfield.get(32767)); assert_value_range(&bitfield, 32760, 7, false); assert_eq!(bitfield.index_of(true, 33), Some(32767)); assert_eq!(bitfield.last_index_of(true, 9), Some(0)); @@ -217,7 +213,7 @@ mod tests { assert_value_range(&bitfield, 5, 59, false); bitfield.set_range(1, 3, false); - assert_eq!(bitfield.get(0), true); + assert!(bitfield.get(0)); assert_value_range(&bitfield, 1, 3, false); assert_value_range(&bitfield, 4, 1, true); assert_value_range(&bitfield, 5, 59, false); diff --git a/src/common/node.rs b/src/common/node.rs index 12b973bd..d47a664a 100644 --- a/src/common/node.rs +++ b/src/common/node.rs @@ -42,7 +42,7 @@ impl Node { Self { index, hash, - length: length as u64, + length, parent: flat_tree::parent(index), data: Some(Vec::with_capacity(0)), blank, @@ -75,7 +75,7 @@ impl NodeTrait for Node { #[inline] fn len(&self) -> u64 { - self.length as u64 + self.length } #[inline] @@ -139,7 +139,7 @@ impl From> for Node { Node { index: partial.index(), parent: partial.parent, - length: partial.len() as u64, + length: partial.len(), hash, data, blank, diff --git a/src/common/peer.rs b/src/common/peer.rs index fb9bc333..ecd291a6 100644 --- a/src/common/peer.rs +++ b/src/common/peer.rs @@ -60,7 +60,7 @@ pub struct ValuelessProof { } impl ValuelessProof { - pub fn into_proof(&mut self, block_value: Option>) -> Proof { + pub fn into_proof(mut self, block_value: Option>) -> Proof { let block = self.block.take().map(|block| DataBlock { index: block.index, nodes: block.nodes, diff --git a/src/core.rs b/src/core.rs index 61a561ec..a46311fe 100644 --- a/src/core.rs +++ b/src/core.rs @@ -176,11 +176,11 @@ where } if let Some(bitfield_update) = &entry.bitfield { - bitfield.update(&bitfield_update); + bitfield.update(bitfield_update); update_contiguous_length( &mut oplog_open_outcome.header, &bitfield, - &bitfield_update, + bitfield_update, ); } if let Some(tree_upgrade) = &entry.tree_upgrade { @@ -272,7 +272,7 @@ where for data in batch.iter() { batch_length += changeset.append(data); } - changeset.hash_and_sign(&self.key_pair.public, &secret_key); + changeset.hash_and_sign(&self.key_pair.public, secret_key); // Write the received data to the block store let info = self @@ -425,7 +425,7 @@ where seek: Option, upgrade: Option, ) -> Result, HypercoreError> { - let mut valueless_proof = self + let valueless_proof = self .create_valueless_proof(block, hash, seek, upgrade) .await?; let value: Option> = if let Some(block) = valueless_proof.block.as_ref() { @@ -532,7 +532,7 @@ where #[instrument(err, skip(self))] pub async fn missing_nodes(&mut self, merkle_tree_index: u64) -> Result { match self.tree.missing_nodes(merkle_tree_index, None)? { - Either::Right(value) => return Ok(value), + Either::Right(value) => Ok(value), Either::Left(instructions) => { let mut instructions = instructions; let mut infos: Vec = vec![]; @@ -557,7 +557,7 @@ where initial_infos: Option<&[StoreInfo]>, ) -> Result { match self.tree.byte_range(index, initial_infos)? { - Either::Right(value) => return Ok(value), + Either::Right(value) => Ok(value), Either::Left(instructions) => { let mut instructions = instructions; let mut infos: Vec = vec![]; @@ -590,7 +590,7 @@ where upgrade.as_ref(), None, )? { - Either::Right(value) => return Ok(value), + Either::Right(value) => Ok(value), Either::Left(instructions) => { let mut instructions = instructions; let mut infos: Vec = vec![]; @@ -627,11 +627,9 @@ where .verify_proof(proof, &self.key_pair.public, Some(&infos))? { Either::Right(value) => Ok(value), - Either::Left(_) => { - return Err(HypercoreError::InvalidOperation { - context: format!("Could not verify proof from tree"), - }); - } + Either::Left(_) => Err(HypercoreError::InvalidOperation { + context: "Could not verify proof from tree".to_string(), + }), } } } @@ -671,12 +669,10 @@ fn update_contiguous_length( if c <= end && c > bitfield_update.start { c = bitfield_update.start; } - } else { - if c <= end && c >= bitfield_update.start { - c = end; - while bitfield.get(c) { - c += 1; - } + } else if c <= end && c >= bitfield_update.start { + c = end; + while bitfield.get(c) { + c += 1; } } @@ -1057,14 +1053,14 @@ mod tests { length: u64, ) -> Result, HypercoreError> { let key_pair = generate_keypair(); - Ok(create_hypercore_with_data_and_key_pair( + create_hypercore_with_data_and_key_pair( length, PartialKeypair { public: key_pair.public, secret: Some(key_pair.secret), }, ) - .await?) + .await } async fn create_hypercore_with_data_and_key_pair( diff --git a/src/crypto/hash.rs b/src/crypto/hash.rs index 037f629e..e76c3b2e 100644 --- a/src/crypto/hash.rs +++ b/src/crypto/hash.rs @@ -53,7 +53,7 @@ impl Hash { (right, left) }; - let size = u64_as_be((node1.length + node2.length) as u64); + let size = u64_as_be(node1.length + node2.length); let mut hasher = Blake2b::new(32); hasher.update(&PARENT_TYPE); @@ -85,8 +85,8 @@ impl Hash { for node in roots { let node = node.as_ref(); hasher.update(node.hash()); - hasher.update(&u64_as_be((node.index()) as u64)); - hasher.update(&u64_as_be((node.len()) as u64)); + hasher.update(&u64_as_be(node.index())); + hasher.update(&u64_as_be(node.len())); } Self { @@ -130,7 +130,7 @@ impl Hash { let (mut state, mut size) = State::new_with_size(8); state - .encode_u64((node1.length + node2.length) as u64, &mut size) + .encode_u64(node1.length + node2.length, &mut size) .expect("Encoding u64 should not fail"); let mut hasher = Blake2b::new(32); @@ -153,10 +153,10 @@ impl Hash { let node = node.as_ref(); let (mut state, mut buffer) = State::new_with_size(16); state - .encode_u64(node.index() as u64, &mut buffer) + .encode_u64(node.index(), &mut buffer) .expect("Encoding u64 should not fail"); state - .encode_u64(node.len() as u64, &mut buffer) + .encode_u64(node.len(), &mut buffer) .expect("Encoding u64 should not fail"); hasher.update(node.hash()); @@ -198,7 +198,7 @@ pub fn signable_tree(hash: &[u8], length: u64, fork: u64) -> Box<[u8]> { .encode_fixed_32(&TREE, &mut buffer) .expect("Should be able "); state - .encode_fixed_32(&hash, &mut buffer) + .encode_fixed_32(hash, &mut buffer) .expect("Encoding fixed 32 bytes should not fail"); state .encode_u64(length, &mut buffer) @@ -218,7 +218,7 @@ mod tests { fn hash_with_extra_byte(data: &[u8], byte: u8) -> Box<[u8]> { let mut hasher = Blake2b::new(32); - hasher.update(&data); + hasher.update(data); hasher.update(&[byte]); let hash = hasher.finalize(); hash.as_bytes().into() @@ -337,7 +337,7 @@ mod tests { hasher.update(&HYPERCORE); let hash = hasher.finalize(); let ns = hash.as_bytes(); - let tree: Box<[u8]> = { hash_with_extra_byte(ns, 0).into() }; + let tree: Box<[u8]> = { hash_with_extra_byte(ns, 0) }; assert_eq!(tree, TREE.into()); } } diff --git a/src/data/mod.rs b/src/data/mod.rs index c8fb3889..ad37454a 100644 --- a/src/data/mod.rs +++ b/src/data/mod.rs @@ -20,7 +20,7 @@ impl BlockStore { } pub fn put(&self, value: &[u8], offset: u64) -> StoreInfo { - StoreInfo::new_content(Store::Data, offset, &value.to_vec()) + StoreInfo::new_content(Store::Data, offset, value) } pub fn read( diff --git a/src/encoding.rs b/src/encoding.rs index 6960ad9a..8f715fd1 100644 --- a/src/encoding.rs +++ b/src/encoding.rs @@ -10,6 +10,13 @@ use crate::{ /// Wrapper struct for compact_encoding::State pub struct HypercoreState(pub State); +impl Default for HypercoreState { + /// Passthrought to compact_encoding + fn default() -> Self { + Self::new() + } +} + impl HypercoreState { /// Passthrought to compact_encoding pub fn new() -> HypercoreState { @@ -72,7 +79,7 @@ impl CompactEncoding> for HypercoreState { fn preencode(&mut self, value: &Vec) -> Result { let len = value.len(); self.0.preencode(&len)?; - for val in value.into_iter() { + for val in value { self.preencode(val)?; } Ok(self.end()) diff --git a/src/oplog/entry.rs b/src/oplog/entry.rs index bdff2353..e86f4d10 100644 --- a/src/oplog/entry.rs +++ b/src/oplog/entry.rs @@ -86,10 +86,10 @@ pub struct Entry { impl CompactEncoding for HypercoreState { fn preencode(&mut self, value: &Entry) -> Result { self.0.add_end(1)?; // flags - if value.user_data.len() > 0 { + if !value.user_data.is_empty() { self.0.preencode(&value.user_data)?; } - if value.tree_nodes.len() > 0 { + if !value.tree_nodes.is_empty() { self.preencode(&value.tree_nodes)?; } if let Some(tree_upgrade) = &value.tree_upgrade { @@ -105,20 +105,20 @@ impl CompactEncoding for HypercoreState { let start = self.0.start(); self.0.add_start(1)?; let mut flags: u8 = 0; - if value.user_data.len() > 0 { - flags = flags | 1; + if !value.user_data.is_empty() { + flags |= 1; self.0.encode(&value.user_data, buffer)?; } - if value.tree_nodes.len() > 0 { - flags = flags | 2; + if !value.tree_nodes.is_empty() { + flags |= 2; self.encode(&value.tree_nodes, buffer)?; } if let Some(tree_upgrade) = &value.tree_upgrade { - flags = flags | 4; + flags |= 4; self.encode(tree_upgrade, buffer)?; } if let Some(bitfield) = &value.bitfield { - flags = flags | 8; + flags |= 8; self.encode(bitfield, buffer)?; } diff --git a/src/storage/mod.rs b/src/storage/mod.rs index 971ad675..1aafb248 100644 --- a/src/storage/mod.rs +++ b/src/storage/mod.rs @@ -34,7 +34,7 @@ impl Clone for PartialKeypair { None => None, }; PartialKeypair { - public: self.public.clone(), + public: self.public, secret, } } @@ -233,7 +233,7 @@ where if !info.miss { if let Some(data) = &info.data { storage - .write(info.index, &data.to_vec()) + .write(info.index, data) .await .map_err(map_random_access_err)?; } @@ -278,7 +278,7 @@ impl Storage { pub async fn new_memory() -> Result { let create = |_| async { Ok(RandomAccessMemory::default()) }.boxed(); // No reason to overwrite, as this is a new memory segment - Ok(Self::open(create, false).await?) + Self::open(create, false).await } } @@ -296,6 +296,6 @@ impl Storage { }; RandomAccessDisk::open(dir.as_path().join(name)).boxed() }; - Ok(Self::open(storage, overwrite).await?) + Self::open(storage, overwrite).await } } diff --git a/src/tree/merkle_tree.rs b/src/tree/merkle_tree.rs index 6f0bf260..03003303 100644 --- a/src/tree/merkle_tree.rs +++ b/src/tree/merkle_tree.rs @@ -90,7 +90,7 @@ impl MerkleTree { roots.push(node); } if length > 0 { - length = length / 2; + length /= 2; } let signature: Option = if header_tree.signature.len() > 0 { Some( @@ -259,7 +259,7 @@ impl MerkleTree { for i in 0..r { tree_offset += self.roots[i].length; } - return Ok(Either::Right(tree_offset as u64)); + return Ok(Either::Right(tree_offset)); } parent.index } else { @@ -289,16 +289,15 @@ impl MerkleTree { let mut changeset = self.changeset(); let mut instructions: Vec = Vec::new(); - for i in 0..full_roots.len() { - let root = full_roots[i]; - if i < changeset.roots.len() && changeset.roots[i].index == root { + for (i, root) in full_roots.iter().enumerate() { + if i < changeset.roots.len() && changeset.roots[i].index == *root { continue; } while changeset.roots.len() > i { changeset.roots.pop(); } - let node_or_instruction = self.required_node(root, &nodes)?; + let node_or_instruction = self.required_node(*root, &nodes)?; match node_or_instruction { Either::Left(instruction) => { instructions.push(instruction); @@ -467,14 +466,10 @@ impl MerkleTree { }; let data_seek: Option = if let Some(seek) = seek.as_ref() { - if let Some(p_seek) = p.seek { - Some(DataSeek { - bytes: seek.bytes, - nodes: p_seek, - }) - } else { - None - } + p.seek.map(|p_seek| DataSeek { + bytes: seek.bytes, + nodes: p_seek, + }) } else { None }; @@ -882,8 +877,7 @@ impl MerkleTree { flat_tree::full_roots(head, &mut roots); let mut bytes = bytes; - for i in 0..roots.len() { - let root = roots[i]; + for root in roots { let node_or_instruction = self.required_node(root, nodes)?; match node_or_instruction { Either::Left(instruction) => { @@ -1043,11 +1037,8 @@ impl MerkleTree { if is_seek && iter.contains(seek_root) && iter.index() != seek_root { let success_or_instruction = self.seek_proof(seek_root, iter.index(), p, nodes)?; - match success_or_instruction { - Either::Left(new_instructions) => { - instructions.extend(new_instructions); - } - _ => (), + if let Either::Left(new_instructions) = success_or_instruction { + instructions.extend(new_instructions); } } else { let node_or_instruction = self.required_node(iter.index(), nodes)?; @@ -1416,7 +1407,7 @@ fn verify_upgrade( } else { NodeQueue::new(upgrade.nodes.clone(), None) }; - let mut grow: bool = changeset.roots.len() > 0; + let mut grow: bool = !changeset.roots.is_empty(); let mut i: usize = 0; let to: u64 = 2 * (upgrade.start + upgrade.length); let mut iter = flat_tree::Iterator::new(0); @@ -1466,7 +1457,7 @@ fn verify_upgrade( iter.sibling(); } changeset.fork = fork; - changeset.verify_and_set_signature(&upgrade.signature, &public_key)?; + changeset.verify_and_set_signature(&upgrade.signature, public_key)?; Ok(q.extra.is_none()) } @@ -1507,15 +1498,13 @@ fn normalize_indexed( nodes: block.nodes, last_index: block.index, }) - } else if let Some(hash) = hash { - Some(NormalizedIndexed { + } else { + hash.map(|hash| NormalizedIndexed { value: false, index: hash.index, nodes: hash.nodes, last_index: flat_tree::right_span(hash.index) / 2, }) - } else { - None } } @@ -1533,14 +1522,12 @@ fn normalize_data(block: Option<&DataBlock>, hash: Option<&DataHash>) -> Option< index: block.index * 2, nodes: block.nodes.clone(), }) - } else if let Some(hash) = hash { - Some(NormalizedData { + } else { + hash.map(|hash| NormalizedData { value: None, index: hash.index, nodes: hash.nodes.clone(), }) - } else { - None } } diff --git a/src/tree/merkle_tree_changeset.rs b/src/tree/merkle_tree_changeset.rs index 291f104a..57e0e130 100644 --- a/src/tree/merkle_tree_changeset.rs +++ b/src/tree/merkle_tree_changeset.rs @@ -75,7 +75,7 @@ impl MerkleTreeChangeset { let node = Node::new( iter.parent(), - Hash::parent(&a, &b).as_bytes().into(), + Hash::parent(a, b).as_bytes().into(), a.length + b.length, ); let _ = &self.nodes.push(node.clone()); @@ -89,7 +89,7 @@ impl MerkleTreeChangeset { pub fn hash_and_sign(&mut self, public_key: &PublicKey, secret_key: &SecretKey) { let hash = self.hash(); let signable = self.signable(&hash); - let signature = sign(&public_key, &secret_key, &signable); + let signature = sign(public_key, secret_key, &signable); self.hash = Some(hash); self.signature = Some(signature); } @@ -106,7 +106,7 @@ impl MerkleTreeChangeset { context: "Could not parse signature".to_string(), })?; let hash = self.hash(); - verify(&public_key, &self.signable(&hash), Some(&signature))?; + verify(public_key, &self.signable(&hash), Some(&signature))?; // Set values to changeset self.hash = Some(hash); From 6fd5f7408bea51d5bee623268f64c9b42952786b Mon Sep 17 00:00:00 2001 From: Timo Tiuraniemi Date: Wed, 5 Apr 2023 12:03:00 +0300 Subject: [PATCH 122/157] Fix rest of src/ clippy warnings --- src/bitfield/fixed.rs | 14 ++--------- src/oplog/header.rs | 8 +++--- src/oplog/mod.rs | 55 ++++++++++++++++++++--------------------- src/tree/merkle_tree.rs | 5 ++-- 4 files changed, 36 insertions(+), 46 deletions(-) diff --git a/src/bitfield/fixed.rs b/src/bitfield/fixed.rs index 874c18c8..71362293 100644 --- a/src/bitfield/fixed.rs +++ b/src/bitfield/fixed.rs @@ -133,22 +133,12 @@ impl FixedBitfield { /// Finds the first index of the value after given position. Returns None if not found. pub fn index_of(&self, value: bool, position: u32) -> Option { - for i in position..FIXED_BITFIELD_BITS_LENGTH as u32 { - if self.get(i) == value { - return Some(i); - } - } - None + (position..FIXED_BITFIELD_BITS_LENGTH as u32).find(|&i| self.get(i) == value) } /// Finds the last index of the value before given position. Returns None if not found. pub fn last_index_of(&self, value: bool, position: u32) -> Option { - for i in (0..position + 1).rev() { - if self.get(i) == value { - return Some(i); - } - } - None + (0..position + 1).rev().find(|&i| self.get(i) == value) } } diff --git a/src/oplog/header.rs b/src/oplog/header.rs index 482ff8bb..e7429fa6 100644 --- a/src/oplog/header.rs +++ b/src/oplog/header.rs @@ -5,7 +5,7 @@ use crate::PartialKeypair; /// Oplog header. #[derive(Debug, Clone)] -pub struct Header { +pub(crate) struct Header { pub(crate) types: HeaderTypes, // TODO: This is a keyValueArray in JS pub(crate) user_data: Vec, @@ -48,7 +48,7 @@ impl Header { /// Oplog header types #[derive(Debug, PartialEq, Clone)] -pub struct HeaderTypes { +pub(crate) struct HeaderTypes { pub(crate) tree: String, pub(crate) bitfield: String, pub(crate) signer: String, @@ -90,7 +90,7 @@ impl CompactEncoding for State { /// Oplog header tree #[derive(Debug, PartialEq, Clone)] -pub struct HeaderTree { +pub(crate) struct HeaderTree { pub(crate) fork: u64, pub(crate) length: u64, pub(crate) root_hash: Box<[u8]>, @@ -188,7 +188,7 @@ impl CompactEncoding for State { /// Oplog header hints #[derive(Debug, Clone)] -pub struct HeaderHints { +pub(crate) struct HeaderHints { pub(crate) reorgs: Vec, } diff --git a/src/oplog/mod.rs b/src/oplog/mod.rs index 6743a57f..5d7a1c8e 100644 --- a/src/oplog/mod.rs +++ b/src/oplog/mod.rs @@ -9,10 +9,10 @@ use crate::{HypercoreError, Node, PartialKeypair}; mod entry; mod header; -pub use entry::{Entry, EntryTreeUpgrade}; -pub use header::{Header, HeaderTree}; +pub(crate) use entry::{Entry, EntryTreeUpgrade}; +pub(crate) use header::{Header, HeaderTree}; -pub const MAX_OPLOG_ENTRIES_BYTE_SIZE: u64 = 65536; +pub(crate) const MAX_OPLOG_ENTRIES_BYTE_SIZE: u64 = 65536; /// Oplog. /// @@ -20,7 +20,7 @@ pub const MAX_OPLOG_ENTRIES_BYTE_SIZE: u64 = 65536; /// and one is the older. Which one is used depends on the value stored in the eigth byte's /// eight bit of the stored headers. #[derive(Debug)] -pub struct Oplog { +pub(crate) struct Oplog { header_bits: [bool; 2], pub(crate) entries_length: u64, pub(crate) entries_byte_length: u64, @@ -28,14 +28,14 @@ pub struct Oplog { /// Oplog create header outcome #[derive(Debug)] -pub struct OplogCreateHeaderOutcome { +pub(crate) struct OplogCreateHeaderOutcome { pub header: Header, pub infos_to_flush: Box<[StoreInfo]>, } /// Oplog open outcome #[derive(Debug)] -pub struct OplogOpenOutcome { +pub(crate) struct OplogOpenOutcome { pub oplog: Oplog, pub header: Header, pub infos_to_flush: Box<[StoreInfo]>, @@ -83,7 +83,7 @@ const INITIAL_HEADER_BITS: [bool; 2] = [true, false]; impl Oplog { /// Opens an existing Oplog from existing byte buffer or creates a new one. - pub fn open( + pub(crate) fn open( key_pair: &Option, info: Option, ) -> Result, HypercoreError> { @@ -137,8 +137,8 @@ impl Oplog { Box::new([]), ) } else if let Some(key_pair) = key_pair { - // There is nothing in the oplog, start from new given key pair. - Self::new(key_pair.clone())? + // There is nothing in the oplog, start from fresh given key pair. + Self::fresh(key_pair.clone())? } else { // The storage is empty and no key pair given, erroring return Err(HypercoreError::EmptyStorage { @@ -151,21 +151,17 @@ impl Oplog { let mut entry_offset = OplogSlot::Entries as usize; let mut entries: Vec = Vec::new(); let mut partials: Vec = Vec::new(); - loop { - if let Some(mut entry_outcome) = - Self::validate_leader(entry_offset as usize, &existing)? - { - let entry: Entry = entry_outcome.state.decode(&existing)?; - entries.push(entry); - partials.push(entry_outcome.partial_bit); - entry_offset = (*entry_outcome.state).end(); - } else { - break; - } + while let Some(mut entry_outcome) = + Self::validate_leader(entry_offset, &existing)? + { + let entry: Entry = entry_outcome.state.decode(&existing)?; + entries.push(entry); + partials.push(entry_outcome.partial_bit); + entry_offset = (*entry_outcome.state).end(); } // Remove all trailing partial entries - while partials.len() > 0 && partials[partials.len() - 1] { + while !partials.is_empty() && partials[partials.len() - 1] { entries.pop(); } outcome.entries = Some(entries.into_boxed_slice()); @@ -176,7 +172,7 @@ impl Oplog { } /// Appends an upgraded changeset to the Oplog. - pub fn append_changeset( + pub(crate) fn append_changeset( &mut self, changeset: &MerkleTreeChangeset, bitfield_update: Option, @@ -225,7 +221,11 @@ impl Oplog { } /// Clears a segment, returns infos to write to storage. - pub fn clear(&mut self, start: u64, end: u64) -> Result, HypercoreError> { + pub(crate) fn clear( + &mut self, + start: u64, + end: u64, + ) -> Result, HypercoreError> { let entry: Entry = Entry { user_data: vec![], tree_nodes: vec![], @@ -264,8 +264,7 @@ impl Oplog { } let mut buffer = state.create_buffer(); - for i in 0..len { - let entry = &batch[i]; + for (i, entry) in batch.iter().enumerate() { (*state).add_start(8)?; let start = state.start(); let partial_bit: bool = atomic && i < len - 1; @@ -286,7 +285,7 @@ impl Oplog { Ok(vec![StoreInfo::new_content(Store::Oplog, index, &buffer)].into_boxed_slice()) } - fn new(key_pair: PartialKeypair) -> Result { + fn fresh(key_pair: PartialKeypair) -> Result { let entries_length: u64 = 0; let entries_byte_length: u64 = 0; let header = Header::new(key_pair); @@ -318,7 +317,7 @@ impl Oplog { // Get the right slot and header bit let (oplog_slot, header_bit) = Oplog::get_next_header_oplog_slot_and_bit_value(¤t_header_bits); - let mut new_header_bits = current_header_bits.clone(); + let mut new_header_bits = current_header_bits; match oplog_slot { OplogSlot::FirstHeader => new_header_bits[0] = header_bit, OplogSlot::SecondHeader => new_header_bits[1] = header_bit, @@ -393,7 +392,7 @@ impl Oplog { /// `State` for the header/entry that the leader was for. fn validate_leader( index: usize, - buffer: &Box<[u8]>, + buffer: &[u8], ) -> Result, HypercoreError> { if buffer.len() < index + 8 { return Ok(None); diff --git a/src/tree/merkle_tree.rs b/src/tree/merkle_tree.rs index 03003303..01a885d4 100644 --- a/src/tree/merkle_tree.rs +++ b/src/tree/merkle_tree.rs @@ -801,7 +801,7 @@ impl MerkleTree { #[cfg(feature = "cache")] if let Some(node_cache) = &self.node_cache { if let Some(node) = node_cache.get(&index) { - return Ok(Either::Right(Some(node.clone()))); + return Ok(Either::Right(Some(node))); } } @@ -1106,6 +1106,7 @@ impl MerkleTree { } } + #[allow(clippy::too_many_arguments)] fn upgrade_proof( &self, indexed: Option<&NormalizedIndexed>, @@ -1304,7 +1305,7 @@ impl MerkleTree { Some(infos) => { let mut nodes: IntMap> = IntMap::with_capacity(infos.len()); for info in infos { - let index = index_from_info(&info); + let index = index_from_info(info); if !info.miss { let node = node_from_bytes(&index, info.data.as_ref().unwrap())?; #[cfg(feature = "cache")] From 673381ee2b8bb59d9cbdfcac1ec51e73cf3fbf4e Mon Sep 17 00:00:00 2001 From: Timo Tiuraniemi Date: Wed, 5 Apr 2023 12:13:25 +0300 Subject: [PATCH 123/157] Fix clippy from tests and benches --- benches/disk.rs | 1 + benches/memory.rs | 1 + src/builder.rs | 7 +++++++ tests/common/mod.rs | 2 +- tests/js/mod.rs | 4 ++-- tests/js_interop.rs | 4 ++-- tests/model.rs | 2 +- 7 files changed, 15 insertions(+), 6 deletions(-) diff --git a/benches/disk.rs b/benches/disk.rs index 66aa41d3..fe5b46ac 100644 --- a/benches/disk.rs +++ b/benches/disk.rs @@ -77,6 +77,7 @@ fn read_disk(c: &mut Criterion) { }); } +#[allow(clippy::unit_arg)] fn clear_disk(c: &mut Criterion) { let mut group = c.benchmark_group("slow_call"); group.measurement_time(Duration::from_secs(20)); diff --git a/benches/memory.rs b/benches/memory.rs index ee26f9b9..959b7a76 100644 --- a/benches/memory.rs +++ b/benches/memory.rs @@ -71,6 +71,7 @@ fn read_memory(c: &mut Criterion) { }); } +#[allow(clippy::unit_arg)] fn clear_memory(c: &mut Criterion) { c.bench_function("clear_memory", move |b| { b.to_async(AsyncStdExecutor) diff --git a/src/builder.rs b/src/builder.rs index 2b7a5036..4e18dad2 100644 --- a/src/builder.rs +++ b/src/builder.rs @@ -13,6 +13,13 @@ use crate::{core::HypercoreOptions, Hypercore, HypercoreError, PartialKeypair, S #[derive(Debug)] pub struct CacheOptionsBuilder(CacheOptions); +#[cfg(feature = "cache")] +impl Default for CacheOptionsBuilder { + fn default() -> Self { + Self::new() + } +} + #[cfg(feature = "cache")] impl CacheOptionsBuilder { /// Create a CacheOptions builder with default options diff --git a/tests/common/mod.rs b/tests/common/mod.rs index 065a82db..4457bbd4 100644 --- a/tests/common/mod.rs +++ b/tests/common/mod.rs @@ -54,7 +54,7 @@ pub fn hash_file(file: String) -> Option { let hash = format!("{:X}", hash_bytes); // Empty file has this hash, don't make a difference between missing and empty file. Rust // is much easier and performant to write if the empty file is created. - if hash == "E3B0C44298FC1C149AFBF4C8996FB92427AE41E4649B934CA495991B7852B855".to_string() { + if hash == *"E3B0C44298FC1C149AFBF4C8996FB92427AE41E4649B934CA495991B7852B855" { None } else { Some(format!("{:X}", hash_bytes)) diff --git a/tests/js/mod.rs b/tests/js/mod.rs index b80730ce..b0da51d4 100644 --- a/tests/js/mod.rs +++ b/tests/js/mod.rs @@ -19,7 +19,7 @@ pub fn cleanup() { pub fn install() { let status = Command::new("npm") .current_dir("tests/js") - .args(&["install"]) + .args(["install"]) .status() .expect("Unable to run npm install"); assert_eq!( @@ -38,7 +38,7 @@ pub fn prepare_test_set(test_set: &str) -> String { pub fn js_run_step(step: u8, test_set: &str) { let status = Command::new("npm") .current_dir("tests/js") - .args(&["run", "step", &step.to_string(), test_set]) + .args(["run", "step", &step.to_string(), test_set]) .status() .expect("Unable to run npm run"); assert_eq!( diff --git a/tests/js_interop.rs b/tests/js_interop.rs index c9d888a6..6773a5da 100644 --- a/tests/js_interop.rs +++ b/tests/js_interop.rs @@ -89,7 +89,7 @@ async fn step_3_read_and_append_unflushed(work_dir: &str) -> Result<()> { let append_outcome = hypercore.append_batch(&[b"second", b"third"]).await?; assert_eq!(append_outcome.length, 5); assert_eq!(append_outcome.byte_length, 26); - let multi_block = &[0x61 as u8; 4096 * 3]; + let multi_block = &[0x61_u8; 4096 * 3]; let append_outcome = hypercore.append(multi_block).await?; assert_eq!(append_outcome.length, 6); assert_eq!(append_outcome.byte_length, 12314); @@ -112,7 +112,7 @@ async fn step_4_append_with_flush(work_dir: &str) -> Result<()> { for i in 0..5 { let append_outcome = hypercore.append(&[i]).await?; assert_eq!(append_outcome.length, (6 + i + 1) as u64); - assert_eq!(append_outcome.byte_length, (12314 + i as u64 + 1) as u64); + assert_eq!(append_outcome.byte_length, (12314 + i as u64 + 1)); } Ok(()) } diff --git a/tests/model.rs b/tests/model.rs index c0b525dd..3b2cf2b3 100644 --- a/tests/model.rs +++ b/tests/model.rs @@ -105,7 +105,7 @@ async fn assert_implementation_matches_model(ops: Vec) -> bool { let start = { let result = model.len() as u64 / len_divisor_for_start as u64; if result == model.len() as u64 { - if model.len() > 0 { + if !model.is_empty() { result - 1 } else { 0 From 910de8c388423d52ba5e9a48f3217230ff070e2c Mon Sep 17 00:00:00 2001 From: Timo Tiuraniemi Date: Wed, 5 Apr 2023 13:03:55 +0300 Subject: [PATCH 124/157] Reintroduce test failures from warnings --- src/lib.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index 28eecf9e..f95fff30 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -2,8 +2,8 @@ #![forbid(rust_2018_idioms, rust_2018_compatibility)] #![forbid(missing_debug_implementations)] #![forbid(missing_docs)] -// FIXME: Off during v10 coding -// #![cfg_attr(test, deny(warnings))] +#![cfg_attr(test, deny(warnings))] +#![doc(test(attr(deny(warnings))))] //! ## Introduction //! Hypercore is a secure, distributed append-only log. Built for sharing From 671d4ac26bec42759319eef2950e4977c1384417 Mon Sep 17 00:00:00 2001 From: Timo Tiuraniemi Date: Thu, 6 Apr 2023 09:44:37 +0300 Subject: [PATCH 125/157] Fix visibility to enable unreachable_pub --- src/bitfield/dynamic.rs | 18 +++++++-------- src/bitfield/fixed.rs | 18 +++++++-------- src/bitfield/mod.rs | 2 +- src/common/mod.rs | 8 ++++--- src/common/node.rs | 2 +- src/common/peer.rs | 18 ++++++--------- src/common/store.rs | 24 +++++++++---------- src/core.rs | 4 ++-- src/crypto/hash.rs | 26 ++++++++++++--------- src/crypto/key_pair.rs | 28 +++++++++++++++++++++-- src/crypto/mod.rs | 7 ++---- src/data/mod.rs | 10 ++++---- src/lib.rs | 9 ++++---- src/oplog/entry.rs | 2 +- src/oplog/header.rs | 8 +++---- src/oplog/mod.rs | 18 +++++++-------- src/prelude.rs | 5 ++-- src/storage/mod.rs | 28 +---------------------- src/tree/merkle_tree.rs | 38 +++++++++++++++---------------- src/tree/merkle_tree_changeset.rs | 21 ++++++++++------- src/tree/mod.rs | 4 ++-- 21 files changed, 151 insertions(+), 147 deletions(-) diff --git a/src/bitfield/dynamic.rs b/src/bitfield/dynamic.rs index f5c68894..6c827c47 100644 --- a/src/bitfield/dynamic.rs +++ b/src/bitfield/dynamic.rs @@ -13,14 +13,14 @@ const DYNAMIC_BITFIELD_PAGE_SIZE: usize = 32768; /// https://github.com/hypercore-protocol/hypercore/blob/master/lib/bitfield.js /// for reference. #[derive(Debug)] -pub struct DynamicBitfield { +pub(crate) struct DynamicBitfield { pages: intmap::IntMap>, biggest_page_index: u64, unflushed: Vec, } impl DynamicBitfield { - pub fn open(info: Option) -> Either { + pub(crate) fn open(info: Option) -> Either { match info { None => Either::Left(StoreInfoInstruction::new_size(Store::Bitfield, 0)), Some(info) => { @@ -68,7 +68,7 @@ impl DynamicBitfield { } /// Flushes pending changes, returns info slices to write to storage. - pub fn flush(&mut self) -> Box<[StoreInfo]> { + pub(crate) fn flush(&mut self) -> Box<[StoreInfo]> { let mut infos_to_flush: Vec = Vec::with_capacity(self.unflushed.len()); for unflushed_id in &self.unflushed { let mut p = self.pages.get_mut(*unflushed_id).unwrap().borrow_mut(); @@ -84,7 +84,7 @@ impl DynamicBitfield { infos_to_flush.into_boxed_slice() } - pub fn get(&self, index: u64) -> bool { + pub(crate) fn get(&self, index: u64) -> bool { let j = index & (DYNAMIC_BITFIELD_PAGE_SIZE as u64 - 1); let i = (index - j) / DYNAMIC_BITFIELD_PAGE_SIZE as u64; @@ -97,7 +97,7 @@ impl DynamicBitfield { } #[allow(dead_code)] - pub fn set(&mut self, index: u64, value: bool) -> bool { + pub(crate) fn set(&mut self, index: u64, value: bool) -> bool { let j = index & (DYNAMIC_BITFIELD_PAGE_SIZE as u64 - 1); let i = (index - j) / DYNAMIC_BITFIELD_PAGE_SIZE as u64; @@ -123,7 +123,7 @@ impl DynamicBitfield { changed } - pub fn update(&mut self, bitfield_update: &BitfieldUpdate) { + pub(crate) fn update(&mut self, bitfield_update: &BitfieldUpdate) { self.set_range( bitfield_update.start, bitfield_update.length, @@ -131,7 +131,7 @@ impl DynamicBitfield { ) } - pub fn set_range(&mut self, start: u64, length: u64, value: bool) { + pub(crate) fn set_range(&mut self, start: u64, length: u64, value: bool) { let mut j = start & (DYNAMIC_BITFIELD_PAGE_SIZE as u64 - 1); let mut i = (start - j) / (DYNAMIC_BITFIELD_PAGE_SIZE as u64); let mut length = length; @@ -167,7 +167,7 @@ impl DynamicBitfield { } /// Finds the first index of the value after given position. Returns None if not found. - pub fn index_of(&self, value: bool, position: u64) -> Option { + pub(crate) fn index_of(&self, value: bool, position: u64) -> Option { let first_index = position & (DYNAMIC_BITFIELD_PAGE_SIZE as u64 - 1); let first_page = (position - first_index) / (DYNAMIC_BITFIELD_PAGE_SIZE as u64); @@ -214,7 +214,7 @@ impl DynamicBitfield { } /// Finds the last index of the value before given position. Returns None if not found. - pub fn last_index_of(&self, value: bool, position: u64) -> Option { + pub(crate) fn last_index_of(&self, value: bool, position: u64) -> Option { let last_index = position & (DYNAMIC_BITFIELD_PAGE_SIZE as u64 - 1); let last_page = (position - last_index) / (DYNAMIC_BITFIELD_PAGE_SIZE as u64); diff --git a/src/bitfield/fixed.rs b/src/bitfield/fixed.rs index 71362293..57ad3b41 100644 --- a/src/bitfield/fixed.rs +++ b/src/bitfield/fixed.rs @@ -13,20 +13,20 @@ use std::convert::TryInto; /// TODO: This has been split into segments on the Javascript side "for improved disk performance": /// https://github.com/hypercore-protocol/hypercore/commit/6392021b11d53041a446e9021c7d79350a052d3d #[derive(Debug)] -pub struct FixedBitfield { +pub(crate) struct FixedBitfield { pub(crate) dirty: bool, bitfield: [u32; FIXED_BITFIELD_LENGTH], } impl FixedBitfield { - pub fn new() -> Self { + pub(crate) fn new() -> Self { Self { dirty: false, bitfield: [0; FIXED_BITFIELD_LENGTH], } } - pub fn from_data(data_index: usize, data: &[u8]) -> Self { + pub(crate) fn from_data(data_index: usize, data: &[u8]) -> Self { let mut bitfield = [0; FIXED_BITFIELD_LENGTH]; if data.len() >= data_index + 4 { let mut i = data_index; @@ -46,7 +46,7 @@ impl FixedBitfield { } } - pub fn to_bytes(&self) -> Box<[u8]> { + pub(crate) fn to_bytes(&self) -> Box<[u8]> { let mut data: [u8; FIXED_BITFIELD_BYTES_LENGTH] = [0; FIXED_BITFIELD_BYTES_LENGTH]; let mut i = 0; for elem in self.bitfield { @@ -60,7 +60,7 @@ impl FixedBitfield { data.into() } - pub fn get(&self, index: u32) -> bool { + pub(crate) fn get(&self, index: u32) -> bool { let n = FIXED_BITFIELD_BITS_PER_ELEM; let offset = index & (n - 1); let i: usize = ((index - offset) / n) @@ -69,7 +69,7 @@ impl FixedBitfield { self.bitfield[i] & (1 << offset) != 0 } - pub fn set(&mut self, index: u32, value: bool) -> bool { + pub(crate) fn set(&mut self, index: u32, value: bool) -> bool { let n = FIXED_BITFIELD_BITS_PER_ELEM; let offset = index & (n - 1); let i: usize = ((index - offset) / n) @@ -88,7 +88,7 @@ impl FixedBitfield { true } - pub fn set_range(&mut self, start: u32, length: u32, value: bool) -> bool { + pub(crate) fn set_range(&mut self, start: u32, length: u32, value: bool) -> bool { let end: u32 = start + length; let n = FIXED_BITFIELD_BITS_PER_ELEM; @@ -132,12 +132,12 @@ impl FixedBitfield { } /// Finds the first index of the value after given position. Returns None if not found. - pub fn index_of(&self, value: bool, position: u32) -> Option { + pub(crate) fn index_of(&self, value: bool, position: u32) -> Option { (position..FIXED_BITFIELD_BITS_LENGTH as u32).find(|&i| self.get(i) == value) } /// Finds the last index of the value before given position. Returns None if not found. - pub fn last_index_of(&self, value: bool, position: u32) -> Option { + pub(crate) fn last_index_of(&self, value: bool, position: u32) -> Option { (0..position + 1).rev().find(|&i| self.get(i) == value) } } diff --git a/src/bitfield/mod.rs b/src/bitfield/mod.rs index d15d70d5..9daa246c 100644 --- a/src/bitfield/mod.rs +++ b/src/bitfield/mod.rs @@ -1,4 +1,4 @@ mod dynamic; mod fixed; -pub use dynamic::DynamicBitfield as Bitfield; +pub(crate) use dynamic::DynamicBitfield as Bitfield; diff --git a/src/common/mod.rs b/src/common/mod.rs index b43722de..f5fb6baf 100644 --- a/src/common/mod.rs +++ b/src/common/mod.rs @@ -6,12 +6,14 @@ mod peer; mod store; pub use self::error::HypercoreError; -pub use self::node::{Node, NodeByteRange}; +pub use self::node::Node; +pub(crate) use self::node::NodeByteRange; +pub(crate) use self::peer::ValuelessProof; pub use self::peer::{ DataBlock, DataHash, DataSeek, DataUpgrade, Proof, RequestBlock, RequestSeek, RequestUpgrade, - ValuelessProof, }; -pub use self::store::{Store, StoreInfo, StoreInfoInstruction, StoreInfoType}; +pub use self::store::Store; +pub(crate) use self::store::{StoreInfo, StoreInfoInstruction, StoreInfoType}; #[derive(Debug, Clone, PartialEq, Eq)] pub struct BitfieldUpdate { diff --git a/src/common/node.rs b/src/common/node.rs index d47a664a..7e339d37 100644 --- a/src/common/node.rs +++ b/src/common/node.rs @@ -9,7 +9,7 @@ use crate::crypto::Hash; /// Node byte range #[derive(Debug, Clone, PartialEq, Eq)] -pub struct NodeByteRange { +pub(crate) struct NodeByteRange { pub(crate) index: u64, pub(crate) length: u64, } diff --git a/src/common/peer.rs b/src/common/peer.rs index ecd291a6..c71b9818 100644 --- a/src/common/peer.rs +++ b/src/common/peer.rs @@ -45,22 +45,18 @@ pub struct Proof { #[derive(Debug, Clone, PartialEq)] /// Valueless proof generated from corresponding requests -pub struct ValuelessProof { - /// Fork - pub fork: u64, +pub(crate) struct ValuelessProof { + pub(crate) fork: u64, /// Data block. NB: The ValuelessProof struct uses the Hash type because /// the stored binary value is processed externally to the proof. - pub block: Option, - /// Data hash - pub hash: Option, - /// Data seek - pub seek: Option, - /// Data updrade - pub upgrade: Option, + pub(crate) block: Option, + pub(crate) hash: Option, + pub(crate) seek: Option, + pub(crate) upgrade: Option, } impl ValuelessProof { - pub fn into_proof(mut self, block_value: Option>) -> Proof { + pub(crate) fn into_proof(mut self, block_value: Option>) -> Proof { let block = self.block.take().map(|block| DataBlock { index: block.index, nodes: block.nodes, diff --git a/src/common/store.rs b/src/common/store.rs index fa1a7d68..357ebc03 100644 --- a/src/common/store.rs +++ b/src/common/store.rs @@ -24,7 +24,7 @@ impl std::fmt::Display for Store { /// Information type about a store. #[derive(Debug, PartialEq)] -pub enum StoreInfoType { +pub(crate) enum StoreInfoType { /// Read/write content of the store Content, /// Size in bytes of the store. When flushed, truncates to the given index. `data` is `None`. @@ -34,7 +34,7 @@ pub enum StoreInfoType { /// Piece of information about a store. Useful for indicating changes that should be made to random /// access storages or information read from them. #[derive(Debug)] -pub struct StoreInfo { +pub(crate) struct StoreInfo { pub(crate) store: Store, pub(crate) info_type: StoreInfoType, pub(crate) index: u64, @@ -46,7 +46,7 @@ pub struct StoreInfo { } impl StoreInfo { - pub fn new_content(store: Store, index: u64, data: &[u8]) -> Self { + pub(crate) fn new_content(store: Store, index: u64, data: &[u8]) -> Self { Self { store, info_type: StoreInfoType::Content, @@ -57,7 +57,7 @@ impl StoreInfo { } } - pub fn new_content_miss(store: Store, index: u64) -> Self { + pub(crate) fn new_content_miss(store: Store, index: u64) -> Self { Self { store, info_type: StoreInfoType::Content, @@ -68,7 +68,7 @@ impl StoreInfo { } } - pub fn new_delete(store: Store, index: u64, length: u64) -> Self { + pub(crate) fn new_delete(store: Store, index: u64, length: u64) -> Self { Self { store, info_type: StoreInfoType::Content, @@ -79,7 +79,7 @@ impl StoreInfo { } } - pub fn new_truncate(store: Store, index: u64) -> Self { + pub(crate) fn new_truncate(store: Store, index: u64) -> Self { Self { store, info_type: StoreInfoType::Size, @@ -90,7 +90,7 @@ impl StoreInfo { } } - pub fn new_size(store: Store, index: u64, length: u64) -> Self { + pub(crate) fn new_size(store: Store, index: u64, length: u64) -> Self { Self { store, info_type: StoreInfoType::Size, @@ -104,7 +104,7 @@ impl StoreInfo { /// Represents an instruction to obtain information about a store. #[derive(Debug)] -pub struct StoreInfoInstruction { +pub(crate) struct StoreInfoInstruction { pub(crate) store: Store, pub(crate) info_type: StoreInfoType, pub(crate) index: u64, @@ -113,7 +113,7 @@ pub struct StoreInfoInstruction { } impl StoreInfoInstruction { - pub fn new_content(store: Store, index: u64, length: u64) -> Self { + pub(crate) fn new_content(store: Store, index: u64, length: u64) -> Self { Self { store, info_type: StoreInfoType::Content, @@ -123,7 +123,7 @@ impl StoreInfoInstruction { } } - pub fn new_content_allow_miss(store: Store, index: u64, length: u64) -> Self { + pub(crate) fn new_content_allow_miss(store: Store, index: u64, length: u64) -> Self { Self { store, info_type: StoreInfoType::Content, @@ -133,7 +133,7 @@ impl StoreInfoInstruction { } } - pub fn new_all_content(store: Store) -> Self { + pub(crate) fn new_all_content(store: Store) -> Self { Self { store, info_type: StoreInfoType::Content, @@ -143,7 +143,7 @@ impl StoreInfoInstruction { } } - pub fn new_size(store: Store, index: u64) -> Self { + pub(crate) fn new_size(store: Store, index: u64) -> Self { Self { store, info_type: StoreInfoType::Size, diff --git a/src/core.rs b/src/core.rs index a46311fe..fd379d31 100644 --- a/src/core.rs +++ b/src/core.rs @@ -11,10 +11,10 @@ use crate::common::cache::CacheOptions; use crate::{ bitfield::Bitfield, common::{BitfieldUpdate, HypercoreError, NodeByteRange, Proof, StoreInfo, ValuelessProof}, - crypto::generate_keypair, + crypto::{generate_keypair, PartialKeypair}, data::BlockStore, oplog::{Header, Oplog, MAX_OPLOG_ENTRIES_BYTE_SIZE}, - storage::{PartialKeypair, Storage}, + storage::Storage, tree::{MerkleTree, MerkleTreeChangeset}, RequestBlock, RequestSeek, RequestUpgrade, }; diff --git a/src/crypto/hash.rs b/src/crypto/hash.rs index e76c3b2e..d9953d2b 100644 --- a/src/crypto/hash.rs +++ b/src/crypto/hash.rs @@ -1,4 +1,4 @@ -pub use blake2_rfc::blake2b::Blake2bResult; +pub(crate) use blake2_rfc::blake2b::Blake2bResult; use blake2_rfc::blake2b::Blake2b; use byteorder::{BigEndian, WriteBytesExt}; @@ -26,13 +26,14 @@ const TREE: [u8; 32] = [ /// `BLAKE2b` hash. #[derive(Debug, Clone, PartialEq)] -pub struct Hash { +pub(crate) struct Hash { hash: Blake2bResult, } impl Hash { /// Hash a `Leaf` node. - pub fn from_leaf(data: &[u8]) -> Self { + #[allow(dead_code)] + pub(crate) fn from_leaf(data: &[u8]) -> Self { let size = u64_as_be(data.len() as u64); let mut hasher = Blake2b::new(32); @@ -46,7 +47,8 @@ impl Hash { } /// Hash two `Leaf` nodes hashes together to form a `Parent` hash. - pub fn from_hashes(left: &Node, right: &Node) -> Self { + #[allow(dead_code)] + pub(crate) fn from_hashes(left: &Node, right: &Node) -> Self { let (node1, node2) = if left.index <= right.index { (left, right) } else { @@ -68,7 +70,8 @@ impl Hash { /// Hash a public key. Useful to find the key you're looking for on a public /// network without leaking the key itself. - pub fn for_discovery_key(public_key: PublicKey) -> Self { + #[allow(dead_code)] + pub(crate) fn for_discovery_key(public_key: PublicKey) -> Self { let mut hasher = Blake2b::with_key(32, public_key.as_bytes()); hasher.update(&HYPERCORE); Self { @@ -78,7 +81,8 @@ impl Hash { /// Hash a vector of `Root` nodes. // Called `crypto.tree()` in the JS implementation. - pub fn from_roots(roots: &[impl AsRef]) -> Self { + #[allow(dead_code)] + pub(crate) fn from_roots(roots: &[impl AsRef]) -> Self { let mut hasher = Blake2b::new(32); hasher.update(&ROOT_TYPE); @@ -95,7 +99,7 @@ impl Hash { } /// Returns a byte slice of this `Hash`'s contents. - pub fn as_bytes(&self) -> &[u8] { + pub(crate) fn as_bytes(&self) -> &[u8] { self.hash.as_bytes() } @@ -104,7 +108,7 @@ impl Hash { // for v10 that use LE bytes. /// Hash data - pub fn data(data: &[u8]) -> Self { + pub(crate) fn data(data: &[u8]) -> Self { let (mut state, mut size) = State::new_with_size(8); state .encode_u64(data.len() as u64, &mut size) @@ -121,7 +125,7 @@ impl Hash { } /// Hash a parent - pub fn parent(left: &Node, right: &Node) -> Self { + pub(crate) fn parent(left: &Node, right: &Node) -> Self { let (node1, node2) = if left.index <= right.index { (left, right) } else { @@ -145,7 +149,7 @@ impl Hash { } /// Hash a tree - pub fn tree(roots: &[impl AsRef]) -> Self { + pub(crate) fn tree(roots: &[impl AsRef]) -> Self { let mut hasher = Blake2b::new(32); hasher.update(&ROOT_TYPE); @@ -192,7 +196,7 @@ impl DerefMut for Hash { /// Create a signable buffer for tree. This is treeSignable in Javascript. /// See https://github.com/hypercore-protocol/hypercore/blob/70b271643c4e4b1e5ecae5bb579966dfe6361ff3/lib/caps.js#L17 -pub fn signable_tree(hash: &[u8], length: u64, fork: u64) -> Box<[u8]> { +pub(crate) fn signable_tree(hash: &[u8], length: u64, fork: u64) -> Box<[u8]> { let (mut state, mut buffer) = State::new_with_size(80); state .encode_fixed_32(&TREE, &mut buffer) diff --git a/src/crypto/key_pair.rs b/src/crypto/key_pair.rs index ebafcdc8..1cdda656 100644 --- a/src/crypto/key_pair.rs +++ b/src/crypto/key_pair.rs @@ -1,12 +1,36 @@ //! Generate an `Ed25519` keypair. -pub use ed25519_dalek::{ExpandedSecretKey, Keypair, PublicKey, SecretKey, Signature, Verifier}; - +use ed25519_dalek::{ExpandedSecretKey, Keypair, PublicKey, SecretKey, Signature, Verifier}; use rand::rngs::{OsRng, StdRng}; use rand::SeedableRng; use crate::HypercoreError; +/// Key pair where for read-only hypercores the secret key can also be missing. +#[derive(Debug)] +pub struct PartialKeypair { + /// Public key + pub public: PublicKey, + /// Secret key. If None, the hypercore is read-only. + pub secret: Option, +} + +impl Clone for PartialKeypair { + fn clone(&self) -> Self { + let secret: Option = match &self.secret { + Some(secret) => { + let bytes = secret.to_bytes(); + Some(SecretKey::from_bytes(&bytes).unwrap()) + } + None => None, + }; + PartialKeypair { + public: self.public, + secret, + } + } +} + /// Generate a new `Ed25519` key pair. pub fn generate() -> Keypair { let mut rng = StdRng::from_rng(OsRng::default()).unwrap(); diff --git a/src/crypto/mod.rs b/src/crypto/mod.rs index 699759bf..b14f197f 100644 --- a/src/crypto/mod.rs +++ b/src/crypto/mod.rs @@ -3,8 +3,5 @@ mod hash; mod key_pair; -pub use self::hash::signable_tree; -pub use self::hash::Hash; -pub use self::key_pair::{ - generate as generate_keypair, sign, verify, PublicKey, SecretKey, Signature, -}; +pub(crate) use self::hash::{signable_tree, Hash}; +pub use self::key_pair::{generate as generate_keypair, sign, verify, PartialKeypair}; diff --git a/src/data/mod.rs b/src/data/mod.rs index ad37454a..7a76d3e2 100644 --- a/src/data/mod.rs +++ b/src/data/mod.rs @@ -3,10 +3,10 @@ use futures::future::Either; /// Block store #[derive(Debug, Default)] -pub struct BlockStore {} +pub(crate) struct BlockStore {} impl BlockStore { - pub fn append_batch( + pub(crate) fn append_batch( &self, batch: &[&[u8]], batch_length: usize, @@ -19,11 +19,11 @@ impl BlockStore { StoreInfo::new_content(Store::Data, byte_length, &buffer) } - pub fn put(&self, value: &[u8], offset: u64) -> StoreInfo { + pub(crate) fn put(&self, value: &[u8], offset: u64) -> StoreInfo { StoreInfo::new_content(Store::Data, offset, value) } - pub fn read( + pub(crate) fn read( &self, byte_range: &NodeByteRange, info: Option, @@ -40,7 +40,7 @@ impl BlockStore { } /// Clears a segment, returns infos to write to storage. - pub fn clear(&mut self, start: u64, length: u64) -> StoreInfo { + pub(crate) fn clear(&mut self, start: u64, length: u64) -> StoreInfo { StoreInfo::new_delete(Store::Data, start, length) } } diff --git a/src/lib.rs b/src/lib.rs index f95fff30..f5f296c0 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -2,6 +2,7 @@ #![forbid(rust_2018_idioms, rust_2018_compatibility)] #![forbid(missing_debug_implementations)] #![forbid(missing_docs)] +#![warn(unreachable_pub)] #![cfg_attr(test, deny(warnings))] #![doc(test(attr(deny(warnings))))] @@ -46,9 +47,9 @@ pub use crate::common::{ RequestSeek, RequestUpgrade, Store, }; pub use crate::core::Hypercore; -pub use crate::crypto::{generate_keypair, sign, verify, Signature}; -pub use crate::storage::{PartialKeypair, Storage}; +pub use crate::crypto::{generate_keypair, sign, verify, PartialKeypair}; +pub use crate::storage::Storage; pub use ed25519_dalek::{ - ExpandedSecretKey, Keypair, PublicKey, SecretKey, EXPANDED_SECRET_KEY_LENGTH, KEYPAIR_LENGTH, - PUBLIC_KEY_LENGTH, SECRET_KEY_LENGTH, + ExpandedSecretKey, Keypair, PublicKey, SecretKey, Signature, EXPANDED_SECRET_KEY_LENGTH, + KEYPAIR_LENGTH, PUBLIC_KEY_LENGTH, SECRET_KEY_LENGTH, }; diff --git a/src/oplog/entry.rs b/src/oplog/entry.rs index e86f4d10..62e4299b 100644 --- a/src/oplog/entry.rs +++ b/src/oplog/entry.rs @@ -3,7 +3,7 @@ use crate::{common::BitfieldUpdate, Node}; /// Entry tree upgrade #[derive(Debug)] -pub struct EntryTreeUpgrade { +pub(crate) struct EntryTreeUpgrade { pub(crate) fork: u64, pub(crate) ancestors: u64, pub(crate) length: u64, diff --git a/src/oplog/header.rs b/src/oplog/header.rs index e7429fa6..8326df38 100644 --- a/src/oplog/header.rs +++ b/src/oplog/header.rs @@ -1,7 +1,7 @@ use compact_encoding::{CompactEncoding, EncodingError, State}; -use crate::crypto::{PublicKey, SecretKey}; use crate::PartialKeypair; +use crate::{PublicKey, SecretKey}; /// Oplog header. #[derive(Debug, Clone)] @@ -17,7 +17,7 @@ pub(crate) struct Header { impl Header { /// Creates a new Header from given key pair - pub fn new(key_pair: PartialKeypair) -> Self { + pub(crate) fn new(key_pair: PartialKeypair) -> Self { Self { types: HeaderTypes::new(), user_data: vec![], @@ -54,7 +54,7 @@ pub(crate) struct HeaderTypes { pub(crate) signer: String, } impl HeaderTypes { - pub fn new() -> Self { + pub(crate) fn new() -> Self { Self { tree: "blake2b".to_string(), bitfield: "raw".to_string(), @@ -98,7 +98,7 @@ pub(crate) struct HeaderTree { } impl HeaderTree { - pub fn new() -> Self { + pub(crate) fn new() -> Self { Self { fork: 0, length: 0, diff --git a/src/oplog/mod.rs b/src/oplog/mod.rs index 5d7a1c8e..7cb091f8 100644 --- a/src/oplog/mod.rs +++ b/src/oplog/mod.rs @@ -29,21 +29,21 @@ pub(crate) struct Oplog { /// Oplog create header outcome #[derive(Debug)] pub(crate) struct OplogCreateHeaderOutcome { - pub header: Header, - pub infos_to_flush: Box<[StoreInfo]>, + pub(crate) header: Header, + pub(crate) infos_to_flush: Box<[StoreInfo]>, } /// Oplog open outcome #[derive(Debug)] pub(crate) struct OplogOpenOutcome { - pub oplog: Oplog, - pub header: Header, - pub infos_to_flush: Box<[StoreInfo]>, - pub entries: Option>, + pub(crate) oplog: Oplog, + pub(crate) header: Header, + pub(crate) infos_to_flush: Box<[StoreInfo]>, + pub(crate) entries: Option>, } impl OplogOpenOutcome { - pub fn new(oplog: Oplog, header: Header, infos_to_flush: Box<[StoreInfo]>) -> Self { + pub(crate) fn new(oplog: Oplog, header: Header, infos_to_flush: Box<[StoreInfo]>) -> Self { Self { oplog, header, @@ -51,7 +51,7 @@ impl OplogOpenOutcome { entries: None, } } - pub fn from_create_header_outcome( + pub(crate) fn from_create_header_outcome( oplog: Oplog, create_header_outcome: OplogCreateHeaderOutcome, ) -> Self { @@ -240,7 +240,7 @@ impl Oplog { } /// Flushes pending changes, returns infos to write to storage. - pub fn flush(&mut self, header: &Header) -> Result, HypercoreError> { + pub(crate) fn flush(&mut self, header: &Header) -> Result, HypercoreError> { let (new_header_bits, infos_to_flush) = Self::insert_header(header, 0, self.header_bits)?; self.entries_byte_length = 0; self.entries_length = 0; diff --git a/src/prelude.rs b/src/prelude.rs index 142f3648..0dd26ea4 100644 --- a/src/prelude.rs +++ b/src/prelude.rs @@ -1,4 +1,5 @@ //! Convenience wrapper to import all of Hypercore's core. -pub use crate::common::Store; +pub use crate::common::{HypercoreError, Store}; pub use crate::core::Hypercore; -pub use crate::storage::{PartialKeypair, Storage}; +pub use crate::crypto::PartialKeypair; +pub use crate::storage::Storage; diff --git a/src/storage/mod.rs b/src/storage/mod.rs index 1aafb248..b8745438 100644 --- a/src/storage/mod.rs +++ b/src/storage/mod.rs @@ -1,6 +1,5 @@ //! Save data to a desired storage backend. -use ed25519_dalek::{PublicKey, SecretKey}; use futures::future::FutureExt; #[cfg(not(target_arch = "wasm32"))] use random_access_disk::RandomAccessDisk; @@ -15,31 +14,6 @@ use crate::{ HypercoreError, }; -/// Key pair where for read-only hypercores the secret key can also be missing. -#[derive(Debug)] -pub struct PartialKeypair { - /// Public key - pub public: PublicKey, - /// Secret key. If None, the hypercore is read-only. - pub secret: Option, -} - -impl Clone for PartialKeypair { - fn clone(&self) -> Self { - let secret: Option = match &self.secret { - Some(secret) => { - let bytes = secret.to_bytes(); - Some(SecretKey::from_bytes(&bytes).unwrap()) - } - None => None, - }; - PartialKeypair { - public: self.public, - secret, - } - } -} - /// Save data to a desired storage backend. #[derive(Debug)] pub struct Storage @@ -52,7 +26,7 @@ where oplog: T, } -pub fn map_random_access_err(err: RandomAccessError) -> HypercoreError { +pub(crate) fn map_random_access_err(err: RandomAccessError) -> HypercoreError { match err { RandomAccessError::IO { return_code, diff --git a/src/tree/merkle_tree.rs b/src/tree/merkle_tree.rs index 01a885d4..4c4c36ce 100644 --- a/src/tree/merkle_tree.rs +++ b/src/tree/merkle_tree.rs @@ -9,11 +9,11 @@ use std::convert::TryFrom; #[cfg(feature = "cache")] use crate::common::cache::CacheOptions; use crate::common::{HypercoreError, NodeByteRange, Proof, ValuelessProof}; -use crate::crypto::{Hash, PublicKey}; +use crate::crypto::Hash; use crate::oplog::HeaderTree; use crate::{ common::{StoreInfo, StoreInfoInstruction}, - Node, + Node, PublicKey, }; use crate::{ DataBlock, DataHash, DataSeek, DataUpgrade, RequestBlock, RequestSeek, RequestUpgrade, Store, @@ -24,7 +24,7 @@ use super::MerkleTreeChangeset; /// Merkle tree. /// See https://github.com/hypercore-protocol/hypercore/blob/master/lib/merkle-tree.js #[derive(Debug)] -pub struct MerkleTree { +pub(crate) struct MerkleTree { pub(crate) roots: Vec, pub(crate) length: u64, pub(crate) byte_length: u64, @@ -165,7 +165,7 @@ impl MerkleTree { } /// Get storage byte range of given hypercore index - pub fn byte_range( + pub(crate) fn byte_range( &mut self, hypercore_index: u64, infos: Option<&[StoreInfo]>, @@ -644,7 +644,7 @@ impl MerkleTree { } } - pub fn flush_truncation(&mut self) -> Vec { + pub(crate) fn flush_truncation(&mut self) -> Vec { let offset = if self.truncate_to == 0 { 0 } else { @@ -655,7 +655,7 @@ impl MerkleTree { vec![StoreInfo::new_truncate(Store::Tree, offset)] } - pub fn flush_nodes(&mut self) -> Vec { + pub(crate) fn flush_nodes(&mut self) -> Vec { let mut infos_to_flush: Vec = Vec::with_capacity(self.unflushed.len()); for (_, node) in self.unflushed.drain() { let (mut state, mut buffer) = State::new_with_size(40); @@ -1482,10 +1482,10 @@ fn node_from_bytes(index: &u64, data: &[u8]) -> Result { #[derive(Debug, Copy, Clone)] struct NormalizedIndexed { - pub value: bool, - pub index: u64, - pub nodes: u64, - pub last_index: u64, + value: bool, + index: u64, + nodes: u64, + last_index: u64, } fn normalize_indexed( @@ -1511,9 +1511,9 @@ fn normalize_indexed( #[derive(Debug, Clone)] struct NormalizedData { - pub value: Option>, - pub index: u64, - pub nodes: Vec, + value: Option>, + index: u64, + nodes: Vec, } fn normalize_data(block: Option<&DataBlock>, hash: Option<&DataHash>) -> Option { @@ -1535,10 +1535,10 @@ fn normalize_data(block: Option<&DataBlock>, hash: Option<&DataHash>) -> Option< /// Struct to use for local building of proof #[derive(Debug, Clone)] struct LocalProof { - pub seek: Option>, - pub nodes: Option>, - pub upgrade: Option>, - pub additional_upgrade: Option>, + seek: Option>, + nodes: Option>, + upgrade: Option>, + additional_upgrade: Option>, } fn nodes_to_root(index: u64, nodes: u64, head: u64) -> Result { @@ -1581,7 +1581,7 @@ struct NodeQueue { length: usize, } impl NodeQueue { - pub fn new(nodes: Vec, extra: Option) -> Self { + fn new(nodes: Vec, extra: Option) -> Self { let length = nodes.len() + if extra.is_some() { 1 } else { 0 }; Self { i: 0, @@ -1590,7 +1590,7 @@ impl NodeQueue { length, } } - pub fn shift(&mut self, index: u64) -> Result { + fn shift(&mut self, index: u64) -> Result { if let Some(extra) = self.extra.take() { if extra.index == index { self.length -= 1; diff --git a/src/tree/merkle_tree_changeset.rs b/src/tree/merkle_tree_changeset.rs index 57e0e130..4a9c96d1 100644 --- a/src/tree/merkle_tree_changeset.rs +++ b/src/tree/merkle_tree_changeset.rs @@ -13,7 +13,7 @@ use crate::{ /// This is called "MerkleTreeBatch" in Javascript, see: /// https://github.com/hypercore-protocol/hypercore/blob/master/lib/merkle-tree.js #[derive(Debug)] -pub struct MerkleTreeChangeset { +pub(crate) struct MerkleTreeChangeset { pub(crate) length: u64, pub(crate) ancestors: u64, pub(crate) byte_length: u64, @@ -31,7 +31,12 @@ pub struct MerkleTreeChangeset { } impl MerkleTreeChangeset { - pub fn new(length: u64, byte_length: u64, fork: u64, roots: Vec) -> MerkleTreeChangeset { + pub(crate) fn new( + length: u64, + byte_length: u64, + fork: u64, + roots: Vec, + ) -> MerkleTreeChangeset { Self { length, ancestors: length, @@ -48,7 +53,7 @@ impl MerkleTreeChangeset { } } - pub fn append(&mut self, data: &[u8]) -> usize { + pub(crate) fn append(&mut self, data: &[u8]) -> usize { let len = data.len(); let head = self.length * 2; let mut iter = flat_tree::Iterator::new(head); @@ -58,7 +63,7 @@ impl MerkleTreeChangeset { len } - pub fn append_root(&mut self, node: Node, iter: &mut flat_tree::Iterator) { + pub(crate) fn append_root(&mut self, node: Node, iter: &mut flat_tree::Iterator) { self.upgraded = true; self.length += iter.factor() / 2; self.byte_length += node.length; @@ -86,7 +91,7 @@ impl MerkleTreeChangeset { } /// Hashes and signs the changeset - pub fn hash_and_sign(&mut self, public_key: &PublicKey, secret_key: &SecretKey) { + pub(crate) fn hash_and_sign(&mut self, public_key: &PublicKey, secret_key: &SecretKey) { let hash = self.hash(); let signable = self.signable(&hash); let signature = sign(public_key, secret_key, &signable); @@ -95,7 +100,7 @@ impl MerkleTreeChangeset { } /// Verify and set signature with given public key - pub fn verify_and_set_signature( + pub(crate) fn verify_and_set_signature( &mut self, signature: &[u8], public_key: &PublicKey, @@ -115,12 +120,12 @@ impl MerkleTreeChangeset { } /// Calculates a hash of the current set of roots - pub fn hash(&self) -> Box<[u8]> { + pub(crate) fn hash(&self) -> Box<[u8]> { Hash::tree(&self.roots).as_bytes().into() } /// Creates a signable slice from given hash - pub fn signable(&self, hash: &[u8]) -> Box<[u8]> { + pub(crate) fn signable(&self, hash: &[u8]) -> Box<[u8]> { signable_tree(hash, self.length, self.fork) } } diff --git a/src/tree/mod.rs b/src/tree/mod.rs index d1cea59d..02367a2a 100644 --- a/src/tree/mod.rs +++ b/src/tree/mod.rs @@ -1,5 +1,5 @@ mod merkle_tree; mod merkle_tree_changeset; -pub use merkle_tree::MerkleTree; -pub use merkle_tree_changeset::MerkleTreeChangeset; +pub(crate) use merkle_tree::MerkleTree; +pub(crate) use merkle_tree_changeset::MerkleTreeChangeset; From bffe41c9ccd9f59e55bb4651353e926848cdb0ea Mon Sep 17 00:00:00 2001 From: Timo Tiuraniemi Date: Thu, 6 Apr 2023 11:51:43 +0300 Subject: [PATCH 126/157] Use released compact-encoding --- Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index b938e304..575a441f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -27,7 +27,7 @@ byteorder = "1.3.4" ed25519-dalek = "1.0.1" thiserror = "1" tracing = "0.1" -compact-encoding = { git = "https://github.com/datrs/compact-encoding", branch = "main" } +compact-encoding = "1" flat-tree = { git = "https://github.com/ttiurani/flat-tree", branch = "v10" } lazy_static = "1.4.0" memory-pager = "0.9.0" From 42d7089083504fa0130aa5f9464a6427a47bb31d Mon Sep 17 00:00:00 2001 From: Timo Tiuraniemi Date: Thu, 6 Apr 2023 13:52:33 +0300 Subject: [PATCH 127/157] Use latest random-access-disk with async-std as default --- Cargo.toml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 575a441f..f335fc49 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -47,7 +47,7 @@ intmap = "2.0.0" moka = { version = "0.10.0", optional = true } [target.'cfg(not(target_arch = "wasm32"))'.dependencies] -random-access-disk = { git = "https://github.com/ttiurani/random-access-disk", branch = "thiserror" } +random-access-disk = { git = "https://github.com/ttiurani/random-access-disk", branch = "thiserror", default-features = false } [dev-dependencies] anyhow = "1.0.70" @@ -63,7 +63,8 @@ test-log = { version = "0.2.11", default-features = false, features = ["trace"] tracing-subscriber = { version = "0.3.16", features = ["env-filter", "fmt"] } [features] -default = ["async-std"] +default = ["async-std", "sparse"] +sparse = ["random-access-disk/sparse"] tokio = ["random-access-disk/tokio"] async-std = ["random-access-disk/async-std"] cache = ["moka"] From aea79092a935ba11b01446e3e4242019ce386732 Mon Sep 17 00:00:00 2001 From: Timo Tiuraniemi Date: Thu, 6 Apr 2023 14:20:52 +0300 Subject: [PATCH 128/157] Switch from quickcheck to proptest This makes it possible to run tests on Windows, and bump to rand 0.8. --- Cargo.toml | 3 ++- tests/model.rs | 72 +++++++++++++++++++++++--------------------------- 2 files changed, 35 insertions(+), 40 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index f335fc49..0c9b58eb 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -51,7 +51,8 @@ random-access-disk = { git = "https://github.com/ttiurani/random-access-disk", b [dev-dependencies] anyhow = "1.0.70" -quickcheck = "0.9.2" +proptest = "1.1.0" +proptest-derive = "0.2.0" data-encoding = "2.2.0" remove_dir_all = "0.7.0" tempfile = "3.1.0" diff --git a/tests/model.rs b/tests/model.rs index 3b2cf2b3..e6a52fed 100644 --- a/tests/model.rs +++ b/tests/model.rs @@ -1,67 +1,61 @@ pub mod common; -use quickcheck::{quickcheck, Arbitrary, Gen}; -use rand::seq::SliceRandom; -use rand::Rng; -use std::u8; +use proptest::prelude::*; +use proptest::test_runner::FileFailurePersistence; +use proptest_derive::Arbitrary; -const MAX_FILE_SIZE: u64 = 5 * 10; // 5mb +const MAX_FILE_SIZE: u64 = 50000; -#[derive(Clone, Debug)] +#[derive(Clone, Debug, Arbitrary)] enum Op { Get { + #[proptest(strategy(index_strategy))] index: u64, }, Append { + #[proptest(regex(data_regex))] data: Vec, }, Clear { + #[proptest(strategy(divisor_strategy))] len_divisor_for_start: u8, + #[proptest(strategy(divisor_strategy))] len_divisor_for_length: u8, }, } -impl Arbitrary for Op { - fn arbitrary(g: &mut G) -> Self { - let choices = [0, 1, 2]; - match choices.choose(g).expect("Value should exist") { - 0 => { - let index: u64 = g.gen_range(0, MAX_FILE_SIZE); - Op::Get { index } - } - 1 => { - let length: u64 = g.gen_range(0, MAX_FILE_SIZE / 3); - let mut data = Vec::with_capacity(length as usize); - for _ in 0..length { - data.push(u8::arbitrary(g)); - } - Op::Append { data } - } - 2 => { - let len_divisor_for_start: u8 = g.gen_range(1, 17); - let len_divisor_for_length: u8 = g.gen_range(1, 17); - Op::Clear { - len_divisor_for_start, - len_divisor_for_length, - } - } - err => panic!("Invalid choice {}", err), - } - } +fn index_strategy() -> impl Strategy { + 0..MAX_FILE_SIZE +} + +fn divisor_strategy() -> impl Strategy { + 1_u8..17_u8 } -quickcheck! { +fn data_regex() -> &'static str { + // Write 0..5000 byte chunks of ASCII characters as dummy data + "([ -~]{1,1}\n){0,5000}" +} + +proptest! { + #![proptest_config(ProptestConfig { + failure_persistence: Some(Box::new(FileFailurePersistence::WithSource("regressions"))), + ..Default::default() + })] + + #[test] #[cfg(feature = "async-std")] - fn implementation_matches_model(ops: Vec) -> bool { - async_std::task::block_on(assert_implementation_matches_model(ops)) + fn implementation_matches_model(ops: Vec) { + assert!(async_std::task::block_on(assert_implementation_matches_model(ops))); } + #[test] #[cfg(feature = "tokio")] - fn implementation_matches_model(ops: Vec) -> bool { + fn implementation_matches_model(ops: Vec) { let rt = tokio::runtime::Runtime::new().unwrap(); - rt.block_on(async { + assert!(rt.block_on(async { assert_implementation_matches_model(ops).await - }) + })); } } From b9462e99f752f47a3effb63c42046d3c5caa0464 Mon Sep 17 00:00:00 2001 From: Timo Tiuraniemi Date: Wed, 12 Apr 2023 12:01:45 +0300 Subject: [PATCH 129/157] Use released random access crates --- Cargo.toml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 0c9b58eb..03e88a35 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "hypercore" -version = "0.12.0-alpha.1" +version = "0.12.0-alpha.2" license = "MIT OR Apache-2.0" description = "Secure, distributed, append-only log" documentation = "https://docs.rs/hypercore" @@ -34,8 +34,8 @@ memory-pager = "0.9.0" merkle-tree-stream = "0.12.0" pretty-hash = "0.4.1" rand = "0.7.3" -random-access-memory = { git = "https://github.com/ttiurani/random-access-memory", branch = "thiserror" } -random-access-storage = { git = "https://github.com/ttiurani/random-access-storage" , branch = "thiserror"} +random-access-memory = "3.0.0" +random-access-storage = "5.0.0" sha2 = "0.9.2" sleep-parser = "0.8.0" sparse-bitfield = "0.11.0" @@ -47,7 +47,7 @@ intmap = "2.0.0" moka = { version = "0.10.0", optional = true } [target.'cfg(not(target_arch = "wasm32"))'.dependencies] -random-access-disk = { git = "https://github.com/ttiurani/random-access-disk", branch = "thiserror", default-features = false } +random-access-disk = "3.0.0" [dev-dependencies] anyhow = "1.0.70" @@ -57,7 +57,7 @@ data-encoding = "2.2.0" remove_dir_all = "0.7.0" tempfile = "3.1.0" async-std = { version = "1.12.0", features = ["attributes"] } -tokio = { version = "1.25.0", default-features = false, features = ["macros", "rt", "rt-multi-thread"] } +tokio = { version = "1.27.0", default-features = false, features = ["macros", "rt", "rt-multi-thread"] } sha2 = "0.10.2" criterion = { version = "0.4", features = ["async_std"] } test-log = { version = "0.2.11", default-features = false, features = ["trace"] } From d0ec17bcfd75ad15cc4b835f67c9f4f5403745ff Mon Sep 17 00:00:00 2001 From: Timo Tiuraniemi Date: Wed, 12 Apr 2023 12:05:27 +0300 Subject: [PATCH 130/157] Use released flat-tree crate --- Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index 03e88a35..a8ea6c2e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -28,7 +28,7 @@ ed25519-dalek = "1.0.1" thiserror = "1" tracing = "0.1" compact-encoding = "1" -flat-tree = { git = "https://github.com/ttiurani/flat-tree", branch = "v10" } +flat-tree = "6.0.0" lazy_static = "1.4.0" memory-pager = "0.9.0" merkle-tree-stream = "0.12.0" From c0846677e3a36cfc1fa6629a495acc3ee3ba62f1 Mon Sep 17 00:00:00 2001 From: Timo Tiuraniemi Date: Wed, 12 Apr 2023 12:16:09 +0300 Subject: [PATCH 131/157] Fix accidental default features --- Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index a8ea6c2e..e65101b5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -47,7 +47,7 @@ intmap = "2.0.0" moka = { version = "0.10.0", optional = true } [target.'cfg(not(target_arch = "wasm32"))'.dependencies] -random-access-disk = "3.0.0" +random-access-disk = { version = "3.0.0", default-features = false } [dev-dependencies] anyhow = "1.0.70" From 1a2ff4f6c50f66f05144ddd2a67bab1b25c25ff8 Mon Sep 17 00:00:00 2001 From: Timo Tiuraniemi Date: Tue, 15 Aug 2023 09:44:49 +0300 Subject: [PATCH 132/157] Export structs used in API --- src/core.rs | 8 ++++++++ src/lib.rs | 2 +- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/src/core.rs b/src/core.rs index fd379d31..b009fa6e 100644 --- a/src/core.rs +++ b/src/core.rs @@ -57,17 +57,25 @@ where /// Response from append, matches that of the Javascript result #[derive(Debug)] pub struct AppendOutcome { + /// Length of the hypercore after append pub length: u64, + /// Byte length of the hypercore after append pub byte_length: u64, } /// Info about the hypercore #[derive(Debug)] pub struct Info { + /// Length of the hypercore pub length: u64, + /// Byte length of the hypercore pub byte_length: u64, + /// Continuous length of entries in the hypercore with data + /// starting from index 0 pub contiguous_length: u64, + /// Fork index. 0 if hypercore not forked. pub fork: u64, + /// True if hypercore is writeable, false if read-only pub writeable: bool, } diff --git a/src/lib.rs b/src/lib.rs index f5f296c0..da8b570a 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -46,7 +46,7 @@ pub use crate::common::{ DataBlock, DataHash, DataSeek, DataUpgrade, HypercoreError, Node, Proof, RequestBlock, RequestSeek, RequestUpgrade, Store, }; -pub use crate::core::Hypercore; +pub use crate::core::{AppendOutcome, Hypercore, Info}; pub use crate::crypto::{generate_keypair, sign, verify, PartialKeypair}; pub use crate::storage::Storage; pub use ed25519_dalek::{ From 8326d6d541008d8ae6632aa2c3ffee89dee03f0b Mon Sep 17 00:00:00 2001 From: Timo Tiuraniemi Date: Tue, 15 Aug 2023 10:00:57 +0300 Subject: [PATCH 133/157] Bump version to make new exports usable --- Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index e65101b5..231f3ee0 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "hypercore" -version = "0.12.0-alpha.2" +version = "0.12.0-alpha.3" license = "MIT OR Apache-2.0" description = "Secure, distributed, append-only log" documentation = "https://docs.rs/hypercore" From 0f47cc71e7ec62825136a99c8deb47102f3d13c1 Mon Sep 17 00:00:00 2001 From: Timo Tiuraniemi Date: Tue, 15 Aug 2023 14:36:28 +0300 Subject: [PATCH 134/157] Change append_batch to support both Vec> and &[&[u8]] This gets around the problem of it being very difficult to append a dynamically created batch to the hypercore --- src/core.rs | 19 +++++++++++-------- src/data/mod.rs | 8 ++++---- tests/js_interop.rs | 9 ++++++--- 3 files changed, 21 insertions(+), 15 deletions(-) diff --git a/src/core.rs b/src/core.rs index b009fa6e..5d26d399 100644 --- a/src/core.rs +++ b/src/core.rs @@ -266,26 +266,29 @@ where } /// Appends a given batch of data slices to the hypercore. - #[instrument(err, skip_all, fields(batch_len = batch.len()))] - pub async fn append_batch(&mut self, batch: &[&[u8]]) -> Result { + #[instrument(err, skip_all, fields(batch_len = batch.as_ref().len()))] + pub async fn append_batch, B: AsRef<[A]>>( + &mut self, + batch: B, + ) -> Result { let secret_key = match &self.key_pair.secret { Some(key) => key, None => return Err(HypercoreError::NotWritable), }; - if !batch.is_empty() { + if !batch.as_ref().is_empty() { // Create a changeset for the tree let mut changeset = self.tree.changeset(); let mut batch_length: usize = 0; - for data in batch.iter() { - batch_length += changeset.append(data); + for data in batch.as_ref().iter() { + batch_length += changeset.append(data.as_ref()); } changeset.hash_and_sign(&self.key_pair.public, secret_key); // Write the received data to the block store - let info = self - .block_store - .append_batch(batch, batch_length, self.tree.byte_length); + let info = + self.block_store + .append_batch(batch.as_ref(), batch_length, self.tree.byte_length); self.storage.flush_info(info).await?; // Append the changeset to the Oplog diff --git a/src/data/mod.rs b/src/data/mod.rs index 7a76d3e2..fa70a904 100644 --- a/src/data/mod.rs +++ b/src/data/mod.rs @@ -6,15 +6,15 @@ use futures::future::Either; pub(crate) struct BlockStore {} impl BlockStore { - pub(crate) fn append_batch( + pub(crate) fn append_batch, B: AsRef<[A]>>( &self, - batch: &[&[u8]], + batch: B, batch_length: usize, byte_length: u64, ) -> StoreInfo { let mut buffer: Vec = Vec::with_capacity(batch_length); - for data in batch.iter() { - buffer.extend_from_slice(data); + for data in batch.as_ref().iter() { + buffer.extend_from_slice(data.as_ref()); } StoreInfo::new_content(Store::Data, byte_length, &buffer) } diff --git a/tests/js_interop.rs b/tests/js_interop.rs index 6773a5da..4276c616 100644 --- a/tests/js_interop.rs +++ b/tests/js_interop.rs @@ -71,7 +71,8 @@ async fn step_1_create(work_dir: &str) -> Result<()> { async fn step_2_append_hello_world(work_dir: &str) -> Result<()> { let mut hypercore = open_hypercore(work_dir).await?; - let append_outcome = hypercore.append_batch(&[b"Hello", b"World"]).await?; + let batch: &[&[u8]] = &[b"Hello", b"World"]; + let append_outcome = hypercore.append_batch(batch).await?; assert_eq!(append_outcome.length, 2); assert_eq!(append_outcome.byte_length, 10); Ok(()) @@ -86,14 +87,16 @@ async fn step_3_read_and_append_unflushed(work_dir: &str) -> Result<()> { let append_outcome = hypercore.append(b"first").await?; assert_eq!(append_outcome.length, 3); assert_eq!(append_outcome.byte_length, 15); - let append_outcome = hypercore.append_batch(&[b"second", b"third"]).await?; + let batch: &[&[u8]] = &[b"second", b"third"]; + let append_outcome = hypercore.append_batch(batch).await?; assert_eq!(append_outcome.length, 5); assert_eq!(append_outcome.byte_length, 26); let multi_block = &[0x61_u8; 4096 * 3]; let append_outcome = hypercore.append(multi_block).await?; assert_eq!(append_outcome.length, 6); assert_eq!(append_outcome.byte_length, 12314); - let append_outcome = hypercore.append_batch(&[]).await?; + let batch: Vec> = vec![]; + let append_outcome = hypercore.append_batch(&batch).await?; assert_eq!(append_outcome.length, 6); assert_eq!(append_outcome.byte_length, 12314); let first = hypercore.get(2).await?; From 9c2675f5631ae702d49bb239df2724b8640b96ff Mon Sep 17 00:00:00 2001 From: Timo Tiuraniemi Date: Thu, 17 Aug 2023 14:24:13 +0300 Subject: [PATCH 135/157] Upgrade to ed25519-dalek 2.0.0 and rand 0.8 Also rename generate_keypair to generate_signing_key to match the new type. --- Cargo.toml | 6 ++-- src/core.rs | 16 +++++----- src/crypto/hash.rs | 6 ++-- src/crypto/key_pair.rs | 45 +++++++++------------------- src/crypto/mod.rs | 2 +- src/lib.rs | 6 ++-- src/oplog/header.rs | 50 +++++++++++++++++-------------- src/tree/merkle_tree.rs | 6 ++-- src/tree/merkle_tree_changeset.rs | 8 ++--- tests/common/mod.rs | 6 ++-- 10 files changed, 69 insertions(+), 82 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 231f3ee0..0d0436a4 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "hypercore" -version = "0.12.0-alpha.3" +version = "0.12.0-alpha.4" license = "MIT OR Apache-2.0" description = "Secure, distributed, append-only log" documentation = "https://docs.rs/hypercore" @@ -24,7 +24,7 @@ edition = "2018" # would probably be wise. blake2-rfc = "0.2.18" byteorder = "1.3.4" -ed25519-dalek = "1.0.1" +ed25519-dalek = { version = "2.0.0", features = ["rand_core"] } thiserror = "1" tracing = "0.1" compact-encoding = "1" @@ -33,7 +33,7 @@ lazy_static = "1.4.0" memory-pager = "0.9.0" merkle-tree-stream = "0.12.0" pretty-hash = "0.4.1" -rand = "0.7.3" +rand = "0.8" random-access-memory = "3.0.0" random-access-storage = "5.0.0" sha2 = "0.9.2" diff --git a/src/core.rs b/src/core.rs index 5d26d399..03407cb1 100644 --- a/src/core.rs +++ b/src/core.rs @@ -11,7 +11,7 @@ use crate::common::cache::CacheOptions; use crate::{ bitfield::Bitfield, common::{BitfieldUpdate, HypercoreError, NodeByteRange, Proof, StoreInfo, ValuelessProof}, - crypto::{generate_keypair, PartialKeypair}, + crypto::{generate_signing_key, PartialKeypair}, data::BlockStore, oplog::{Header, Oplog, MAX_OPLOG_ENTRIES_BYTE_SIZE}, storage::Storage, @@ -98,10 +98,10 @@ where None } else { Some(options.key_pair.take().unwrap_or_else(|| { - let key_pair = generate_keypair(); + let signing_key = generate_signing_key(); PartialKeypair { - public: key_pair.public, - secret: Some(key_pair.secret), + public: signing_key.verifying_key(), + secret: Some(signing_key.to_bytes()), } })) }; @@ -283,7 +283,7 @@ where for data in batch.as_ref().iter() { batch_length += changeset.append(data.as_ref()); } - changeset.hash_and_sign(&self.key_pair.public, secret_key); + changeset.hash_and_sign(secret_key); // Write the received data to the block store let info = @@ -1063,12 +1063,12 @@ mod tests { async fn create_hypercore_with_data( length: u64, ) -> Result, HypercoreError> { - let key_pair = generate_keypair(); + let signing_key = generate_signing_key(); create_hypercore_with_data_and_key_pair( length, PartialKeypair { - public: key_pair.public, - secret: Some(key_pair.secret), + public: signing_key.verifying_key(), + secret: Some(signing_key.to_bytes()), }, ) .await diff --git a/src/crypto/hash.rs b/src/crypto/hash.rs index d9953d2b..5d4fba2c 100644 --- a/src/crypto/hash.rs +++ b/src/crypto/hash.rs @@ -3,7 +3,7 @@ pub(crate) use blake2_rfc::blake2b::Blake2bResult; use blake2_rfc::blake2b::Blake2b; use byteorder::{BigEndian, WriteBytesExt}; use compact_encoding::State; -use ed25519_dalek::PublicKey; +use ed25519_dalek::VerifyingKey; use merkle_tree_stream::Node as NodeTrait; use std::convert::AsRef; use std::mem; @@ -71,7 +71,7 @@ impl Hash { /// Hash a public key. Useful to find the key you're looking for on a public /// network without leaking the key itself. #[allow(dead_code)] - pub(crate) fn for_discovery_key(public_key: PublicKey) -> Self { + pub(crate) fn for_discovery_key(public_key: VerifyingKey) -> Self { let mut hasher = Blake2b::with_key(32, public_key.as_bytes()); hasher.update(&HYPERCORE); Self { @@ -282,7 +282,7 @@ mod tests { #[test] fn discovery_key_hashing() -> Result<(), ed25519_dalek::SignatureError> { - let public_key = PublicKey::from_bytes(&[ + let public_key = VerifyingKey::from_bytes(&[ 119, 143, 141, 149, 81, 117, 201, 46, 76, 237, 94, 79, 85, 99, 246, 155, 254, 192, 200, 108, 198, 246, 112, 53, 44, 69, 121, 67, 102, 111, 230, 57, ])?; diff --git a/src/crypto/key_pair.rs b/src/crypto/key_pair.rs index 1cdda656..7767c439 100644 --- a/src/crypto/key_pair.rs +++ b/src/crypto/key_pair.rs @@ -1,50 +1,33 @@ //! Generate an `Ed25519` keypair. -use ed25519_dalek::{ExpandedSecretKey, Keypair, PublicKey, SecretKey, Signature, Verifier}; -use rand::rngs::{OsRng, StdRng}; -use rand::SeedableRng; +use ed25519_dalek::{SecretKey, Signature, Signer, SigningKey, Verifier, VerifyingKey}; +use rand::rngs::OsRng; use crate::HypercoreError; /// Key pair where for read-only hypercores the secret key can also be missing. -#[derive(Debug)] +#[derive(Debug, Clone)] pub struct PartialKeypair { /// Public key - pub public: PublicKey, + pub public: VerifyingKey, /// Secret key. If None, the hypercore is read-only. pub secret: Option, } -impl Clone for PartialKeypair { - fn clone(&self) -> Self { - let secret: Option = match &self.secret { - Some(secret) => { - let bytes = secret.to_bytes(); - Some(SecretKey::from_bytes(&bytes).unwrap()) - } - None => None, - }; - PartialKeypair { - public: self.public, - secret, - } - } -} - /// Generate a new `Ed25519` key pair. -pub fn generate() -> Keypair { - let mut rng = StdRng::from_rng(OsRng::default()).unwrap(); - Keypair::generate(&mut rng) +pub fn generate() -> SigningKey { + let mut csprng = OsRng; + SigningKey::generate(&mut csprng) } /// Sign a byte slice using a keypair's private key. -pub fn sign(public_key: &PublicKey, secret: &SecretKey, msg: &[u8]) -> Signature { - ExpandedSecretKey::from(secret).sign(msg, public_key) +pub fn sign(secret: &SecretKey, msg: &[u8]) -> Signature { + SigningKey::from(secret).sign(msg) } /// Verify a signature on a message with a keypair's public key. pub fn verify( - public: &PublicKey, + public: &VerifyingKey, msg: &[u8], sig: Option<&Signature>, ) -> Result<(), HypercoreError> { @@ -66,9 +49,9 @@ pub fn verify( #[test] fn can_verify_messages() { - let keypair = generate(); + let signing_key = generate(); let from = b"hello"; - let sig = sign(&keypair.public, &keypair.secret, from); - verify(&keypair.public, from, Some(&sig)).unwrap(); - verify(&keypair.public, b"oops", Some(&sig)).unwrap_err(); + let sig = sign(&signing_key.to_bytes(), from); + verify(&signing_key.verifying_key(), from, Some(&sig)).unwrap(); + verify(&signing_key.verifying_key(), b"oops", Some(&sig)).unwrap_err(); } diff --git a/src/crypto/mod.rs b/src/crypto/mod.rs index b14f197f..e9eec459 100644 --- a/src/crypto/mod.rs +++ b/src/crypto/mod.rs @@ -4,4 +4,4 @@ mod hash; mod key_pair; pub(crate) use self::hash::{signable_tree, Hash}; -pub use self::key_pair::{generate as generate_keypair, sign, verify, PartialKeypair}; +pub use self::key_pair::{generate as generate_signing_key, sign, verify, PartialKeypair}; diff --git a/src/lib.rs b/src/lib.rs index da8b570a..16284dee 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -47,9 +47,9 @@ pub use crate::common::{ RequestSeek, RequestUpgrade, Store, }; pub use crate::core::{AppendOutcome, Hypercore, Info}; -pub use crate::crypto::{generate_keypair, sign, verify, PartialKeypair}; +pub use crate::crypto::{generate_signing_key, sign, verify, PartialKeypair}; pub use crate::storage::Storage; pub use ed25519_dalek::{ - ExpandedSecretKey, Keypair, PublicKey, SecretKey, Signature, EXPANDED_SECRET_KEY_LENGTH, - KEYPAIR_LENGTH, PUBLIC_KEY_LENGTH, SECRET_KEY_LENGTH, + SecretKey, Signature, SigningKey, VerifyingKey, KEYPAIR_LENGTH, PUBLIC_KEY_LENGTH, + SECRET_KEY_LENGTH, }; diff --git a/src/oplog/header.rs b/src/oplog/header.rs index 8326df38..5d437e39 100644 --- a/src/oplog/header.rs +++ b/src/oplog/header.rs @@ -1,7 +1,9 @@ use compact_encoding::{CompactEncoding, EncodingError, State}; +use ed25519_dalek::{PUBLIC_KEY_LENGTH, SECRET_KEY_LENGTH}; +use std::convert::TryInto; use crate::PartialKeypair; -use crate::{PublicKey, SecretKey}; +use crate::{SecretKey, VerifyingKey}; /// Oplog header. #[derive(Debug, Clone)] @@ -17,12 +19,12 @@ pub(crate) struct Header { impl Header { /// Creates a new Header from given key pair - pub(crate) fn new(key_pair: PartialKeypair) -> Self { + pub(crate) fn new(signing_key: PartialKeypair) -> Self { Self { types: HeaderTypes::new(), user_data: vec![], tree: HeaderTree::new(), - signer: key_pair, + signer: signing_key, hints: HeaderHints { reorgs: vec![] }, contiguous_length: 0, } @@ -141,11 +143,11 @@ impl CompactEncoding for State { /// maintain binary compatibility, we store the public key in the oplog now twice. impl CompactEncoding for State { fn preencode(&mut self, value: &PartialKeypair) -> Result { - self.add_end(1 + 32)?; + self.add_end(1 + PUBLIC_KEY_LENGTH)?; match &value.secret { Some(_) => { // Also add room for the public key - self.add_end(1 + 64) + self.add_end(1 + SECRET_KEY_LENGTH + PUBLIC_KEY_LENGTH) } None => self.add_end(1), } @@ -160,8 +162,9 @@ impl CompactEncoding for State { self.encode(&public_key_bytes, buffer)?; match &value.secret { Some(secret_key) => { - let mut secret_key_bytes: Vec = Vec::with_capacity(64); - secret_key_bytes.extend_from_slice(secret_key.as_bytes()); + let mut secret_key_bytes: Vec = + Vec::with_capacity(SECRET_KEY_LENGTH + PUBLIC_KEY_LENGTH); + secret_key_bytes.extend_from_slice(secret_key); secret_key_bytes.extend_from_slice(&public_key_bytes); let secret_key_bytes: Box<[u8]> = secret_key_bytes.into_boxed_slice(); self.encode(&secret_key_bytes, buffer) @@ -172,15 +175,19 @@ impl CompactEncoding for State { fn decode(&mut self, buffer: &[u8]) -> Result { let public_key_bytes: Box<[u8]> = self.decode(buffer)?; + let public_key_bytes: [u8; PUBLIC_KEY_LENGTH] = + public_key_bytes[0..PUBLIC_KEY_LENGTH].try_into().unwrap(); let secret_key_bytes: Box<[u8]> = self.decode(buffer)?; - let secret: Option = if secret_key_bytes.len() == 0 { + let secret_key_bytes: [u8; SECRET_KEY_LENGTH] = + secret_key_bytes[0..SECRET_KEY_LENGTH].try_into().unwrap(); + let secret: Option = if secret_key_bytes.is_empty() { None } else { - Some(SecretKey::from_bytes(&secret_key_bytes[0..32]).unwrap()) + Some(secret_key_bytes) }; Ok(PartialKeypair { - public: PublicKey::from_bytes(&public_key_bytes).unwrap(), + public: VerifyingKey::from_bytes(&public_key_bytes).unwrap(), secret, }) } @@ -256,7 +263,7 @@ impl CompactEncoding
for State { mod tests { use super::*; - use crate::crypto::generate_keypair; + use crate::crypto::generate_signing_key; #[test] fn encode_header_types() -> Result<(), EncodingError> { @@ -275,10 +282,10 @@ mod tests { #[test] fn encode_partial_key_pair() -> Result<(), EncodingError> { let mut enc_state = State::new(); - let key_pair = generate_keypair(); + let signing_key = generate_signing_key(); let key_pair = PartialKeypair { - public: key_pair.public, - secret: Some(key_pair.secret), + public: signing_key.verifying_key(), + secret: Some(signing_key.to_bytes()), }; enc_state.preencode(&key_pair)?; let mut buffer = enc_state.create_buffer(); @@ -292,10 +299,7 @@ mod tests { let mut dec_state = State::from_buffer(&buffer); let key_pair_ret: PartialKeypair = dec_state.decode(&buffer)?; assert_eq!(key_pair.public, key_pair_ret.public); - assert_eq!( - key_pair.secret.unwrap().as_bytes(), - key_pair_ret.secret.unwrap().as_bytes() - ); + assert_eq!(key_pair.secret.unwrap(), key_pair_ret.secret.unwrap()); Ok(()) } @@ -315,12 +319,12 @@ mod tests { #[test] fn encode_header() -> Result<(), EncodingError> { let mut enc_state = State::new(); - let key_pair = generate_keypair(); - let key_pair = PartialKeypair { - public: key_pair.public, - secret: Some(key_pair.secret), + let signing_key = generate_signing_key(); + let signing_key = PartialKeypair { + public: signing_key.verifying_key(), + secret: Some(signing_key.to_bytes()), }; - let header = Header::new(key_pair); + let header = Header::new(signing_key); enc_state.preencode(&header)?; let mut buffer = enc_state.create_buffer(); enc_state.encode(&header, &mut buffer)?; diff --git a/src/tree/merkle_tree.rs b/src/tree/merkle_tree.rs index 4c4c36ce..3bb9359d 100644 --- a/src/tree/merkle_tree.rs +++ b/src/tree/merkle_tree.rs @@ -13,7 +13,7 @@ use crate::crypto::Hash; use crate::oplog::HeaderTree; use crate::{ common::{StoreInfo, StoreInfoInstruction}, - Node, PublicKey, + Node, VerifyingKey, }; use crate::{ DataBlock, DataHash, DataSeek, DataUpgrade, RequestBlock, RequestSeek, RequestUpgrade, Store, @@ -509,7 +509,7 @@ impl MerkleTree { pub(crate) fn verify_proof( &mut self, proof: &Proof, - public_key: &PublicKey, + public_key: &VerifyingKey, infos: Option<&[StoreInfo]>, ) -> Result, MerkleTreeChangeset>, HypercoreError> { let nodes: IntMap> = self.infos_to_nodes(infos)?; @@ -1400,7 +1400,7 @@ fn verify_upgrade( fork: u64, upgrade: &DataUpgrade, block_root: Option<&Node>, - public_key: &PublicKey, + public_key: &VerifyingKey, changeset: &mut MerkleTreeChangeset, ) -> Result { let mut q = if let Some(block_root) = block_root { diff --git a/src/tree/merkle_tree_changeset.rs b/src/tree/merkle_tree_changeset.rs index 4a9c96d1..255d0e9a 100644 --- a/src/tree/merkle_tree_changeset.rs +++ b/src/tree/merkle_tree_changeset.rs @@ -1,4 +1,4 @@ -use ed25519_dalek::{PublicKey, SecretKey, Signature}; +use ed25519_dalek::{SecretKey, Signature, VerifyingKey}; use std::convert::TryFrom; use crate::{ @@ -91,10 +91,10 @@ impl MerkleTreeChangeset { } /// Hashes and signs the changeset - pub(crate) fn hash_and_sign(&mut self, public_key: &PublicKey, secret_key: &SecretKey) { + pub(crate) fn hash_and_sign(&mut self, secret_key: &SecretKey) { let hash = self.hash(); let signable = self.signable(&hash); - let signature = sign(public_key, secret_key, &signable); + let signature = sign(secret_key, &signable); self.hash = Some(hash); self.signature = Some(signature); } @@ -103,7 +103,7 @@ impl MerkleTreeChangeset { pub(crate) fn verify_and_set_signature( &mut self, signature: &[u8], - public_key: &PublicKey, + public_key: &VerifyingKey, ) -> Result<(), HypercoreError> { // Verify that the received signature matches the public key let signature = diff --git a/tests/common/mod.rs b/tests/common/mod.rs index 4457bbd4..609f275d 100644 --- a/tests/common/mod.rs +++ b/tests/common/mod.rs @@ -1,4 +1,4 @@ -use ed25519_dalek::{PublicKey, SecretKey, PUBLIC_KEY_LENGTH, SECRET_KEY_LENGTH}; +use ed25519_dalek::{VerifyingKey, PUBLIC_KEY_LENGTH, SECRET_KEY_LENGTH}; use sha2::{Digest, Sha256}; use hypercore::PartialKeypair; @@ -24,8 +24,8 @@ pub struct HypercoreHash { } pub fn get_test_key_pair() -> PartialKeypair { - let public = PublicKey::from_bytes(&TEST_PUBLIC_KEY_BYTES).unwrap(); - let secret = Some(SecretKey::from_bytes(&TEST_SECRET_KEY_BYTES).unwrap()); + let public = VerifyingKey::from_bytes(&TEST_PUBLIC_KEY_BYTES).unwrap(); + let secret = Some(TEST_SECRET_KEY_BYTES); PartialKeypair { public, secret } } From bd9a0e2f0f598bee700c3305b109e25c88e85dd8 Mon Sep 17 00:00:00 2001 From: Timo Tiuraniemi Date: Thu, 17 Aug 2023 15:07:17 +0300 Subject: [PATCH 136/157] Fix bug with storing read-only hypercore --- Cargo.toml | 2 +- src/oplog/header.rs | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 0d0436a4..0bcce643 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "hypercore" -version = "0.12.0-alpha.4" +version = "0.12.0-alpha.5" license = "MIT OR Apache-2.0" description = "Secure, distributed, append-only log" documentation = "https://docs.rs/hypercore" diff --git a/src/oplog/header.rs b/src/oplog/header.rs index 5d437e39..e53431f4 100644 --- a/src/oplog/header.rs +++ b/src/oplog/header.rs @@ -178,11 +178,11 @@ impl CompactEncoding for State { let public_key_bytes: [u8; PUBLIC_KEY_LENGTH] = public_key_bytes[0..PUBLIC_KEY_LENGTH].try_into().unwrap(); let secret_key_bytes: Box<[u8]> = self.decode(buffer)?; - let secret_key_bytes: [u8; SECRET_KEY_LENGTH] = - secret_key_bytes[0..SECRET_KEY_LENGTH].try_into().unwrap(); let secret: Option = if secret_key_bytes.is_empty() { None } else { + let secret_key_bytes: [u8; SECRET_KEY_LENGTH] = + secret_key_bytes[0..SECRET_KEY_LENGTH].try_into().unwrap(); Some(secret_key_bytes) }; From 010f34250e76db9fdc9fac6ddedc2d1dfdfcfd43 Mon Sep 17 00:00:00 2001 From: Timo Tiuraniemi Date: Fri, 18 Aug 2023 08:21:52 +0300 Subject: [PATCH 137/157] For efficiency, use SigningKey directly in PartialKeypair so that it doesn't have to be re-created from SecretKey every time an entry is signed. --- src/core.rs | 4 ++-- src/crypto/key_pair.rs | 10 +++++----- src/oplog/header.rs | 19 +++++++++++-------- src/tree/merkle_tree_changeset.rs | 6 +++--- tests/common/mod.rs | 6 ++++-- 5 files changed, 25 insertions(+), 20 deletions(-) diff --git a/src/core.rs b/src/core.rs index 03407cb1..37cc0244 100644 --- a/src/core.rs +++ b/src/core.rs @@ -101,7 +101,7 @@ where let signing_key = generate_signing_key(); PartialKeypair { public: signing_key.verifying_key(), - secret: Some(signing_key.to_bytes()), + secret: Some(signing_key), } })) }; @@ -1068,7 +1068,7 @@ mod tests { length, PartialKeypair { public: signing_key.verifying_key(), - secret: Some(signing_key.to_bytes()), + secret: Some(signing_key), }, ) .await diff --git a/src/crypto/key_pair.rs b/src/crypto/key_pair.rs index 7767c439..683cb689 100644 --- a/src/crypto/key_pair.rs +++ b/src/crypto/key_pair.rs @@ -1,6 +1,6 @@ //! Generate an `Ed25519` keypair. -use ed25519_dalek::{SecretKey, Signature, Signer, SigningKey, Verifier, VerifyingKey}; +use ed25519_dalek::{Signature, Signer, SigningKey, Verifier, VerifyingKey}; use rand::rngs::OsRng; use crate::HypercoreError; @@ -11,7 +11,7 @@ pub struct PartialKeypair { /// Public key pub public: VerifyingKey, /// Secret key. If None, the hypercore is read-only. - pub secret: Option, + pub secret: Option, } /// Generate a new `Ed25519` key pair. @@ -21,8 +21,8 @@ pub fn generate() -> SigningKey { } /// Sign a byte slice using a keypair's private key. -pub fn sign(secret: &SecretKey, msg: &[u8]) -> Signature { - SigningKey::from(secret).sign(msg) +pub fn sign(signing_key: &SigningKey, msg: &[u8]) -> Signature { + signing_key.sign(msg) } /// Verify a signature on a message with a keypair's public key. @@ -51,7 +51,7 @@ pub fn verify( fn can_verify_messages() { let signing_key = generate(); let from = b"hello"; - let sig = sign(&signing_key.to_bytes(), from); + let sig = sign(&signing_key, from); verify(&signing_key.verifying_key(), from, Some(&sig)).unwrap(); verify(&signing_key.verifying_key(), b"oops", Some(&sig)).unwrap_err(); } diff --git a/src/oplog/header.rs b/src/oplog/header.rs index e53431f4..6596e00b 100644 --- a/src/oplog/header.rs +++ b/src/oplog/header.rs @@ -1,9 +1,9 @@ use compact_encoding::{CompactEncoding, EncodingError, State}; -use ed25519_dalek::{PUBLIC_KEY_LENGTH, SECRET_KEY_LENGTH}; +use ed25519_dalek::{SigningKey, PUBLIC_KEY_LENGTH, SECRET_KEY_LENGTH}; use std::convert::TryInto; use crate::PartialKeypair; -use crate::{SecretKey, VerifyingKey}; +use crate::VerifyingKey; /// Oplog header. #[derive(Debug, Clone)] @@ -164,7 +164,7 @@ impl CompactEncoding for State { Some(secret_key) => { let mut secret_key_bytes: Vec = Vec::with_capacity(SECRET_KEY_LENGTH + PUBLIC_KEY_LENGTH); - secret_key_bytes.extend_from_slice(secret_key); + secret_key_bytes.extend_from_slice(&secret_key.to_bytes()); secret_key_bytes.extend_from_slice(&public_key_bytes); let secret_key_bytes: Box<[u8]> = secret_key_bytes.into_boxed_slice(); self.encode(&secret_key_bytes, buffer) @@ -178,12 +178,12 @@ impl CompactEncoding for State { let public_key_bytes: [u8; PUBLIC_KEY_LENGTH] = public_key_bytes[0..PUBLIC_KEY_LENGTH].try_into().unwrap(); let secret_key_bytes: Box<[u8]> = self.decode(buffer)?; - let secret: Option = if secret_key_bytes.is_empty() { + let secret: Option = if secret_key_bytes.is_empty() { None } else { let secret_key_bytes: [u8; SECRET_KEY_LENGTH] = secret_key_bytes[0..SECRET_KEY_LENGTH].try_into().unwrap(); - Some(secret_key_bytes) + Some(SigningKey::from_bytes(&secret_key_bytes)) }; Ok(PartialKeypair { @@ -285,7 +285,7 @@ mod tests { let signing_key = generate_signing_key(); let key_pair = PartialKeypair { public: signing_key.verifying_key(), - secret: Some(signing_key.to_bytes()), + secret: Some(signing_key), }; enc_state.preencode(&key_pair)?; let mut buffer = enc_state.create_buffer(); @@ -299,7 +299,10 @@ mod tests { let mut dec_state = State::from_buffer(&buffer); let key_pair_ret: PartialKeypair = dec_state.decode(&buffer)?; assert_eq!(key_pair.public, key_pair_ret.public); - assert_eq!(key_pair.secret.unwrap(), key_pair_ret.secret.unwrap()); + assert_eq!( + key_pair.secret.unwrap().to_bytes(), + key_pair_ret.secret.unwrap().to_bytes() + ); Ok(()) } @@ -322,7 +325,7 @@ mod tests { let signing_key = generate_signing_key(); let signing_key = PartialKeypair { public: signing_key.verifying_key(), - secret: Some(signing_key.to_bytes()), + secret: Some(signing_key), }; let header = Header::new(signing_key); enc_state.preencode(&header)?; diff --git a/src/tree/merkle_tree_changeset.rs b/src/tree/merkle_tree_changeset.rs index 255d0e9a..be28873f 100644 --- a/src/tree/merkle_tree_changeset.rs +++ b/src/tree/merkle_tree_changeset.rs @@ -1,4 +1,4 @@ -use ed25519_dalek::{SecretKey, Signature, VerifyingKey}; +use ed25519_dalek::{Signature, SigningKey, VerifyingKey}; use std::convert::TryFrom; use crate::{ @@ -91,10 +91,10 @@ impl MerkleTreeChangeset { } /// Hashes and signs the changeset - pub(crate) fn hash_and_sign(&mut self, secret_key: &SecretKey) { + pub(crate) fn hash_and_sign(&mut self, signing_key: &SigningKey) { let hash = self.hash(); let signable = self.signable(&hash); - let signature = sign(secret_key, &signable); + let signature = sign(signing_key, &signable); self.hash = Some(hash); self.signature = Some(signature); } diff --git a/tests/common/mod.rs b/tests/common/mod.rs index 609f275d..371fa417 100644 --- a/tests/common/mod.rs +++ b/tests/common/mod.rs @@ -1,4 +1,4 @@ -use ed25519_dalek::{VerifyingKey, PUBLIC_KEY_LENGTH, SECRET_KEY_LENGTH}; +use ed25519_dalek::{SigningKey, VerifyingKey, PUBLIC_KEY_LENGTH, SECRET_KEY_LENGTH}; use sha2::{Digest, Sha256}; use hypercore::PartialKeypair; @@ -25,7 +25,9 @@ pub struct HypercoreHash { pub fn get_test_key_pair() -> PartialKeypair { let public = VerifyingKey::from_bytes(&TEST_PUBLIC_KEY_BYTES).unwrap(); - let secret = Some(TEST_SECRET_KEY_BYTES); + let signing_key = SigningKey::from_bytes(&TEST_SECRET_KEY_BYTES); + assert_eq!(public.to_bytes(), signing_key.verifying_key().to_bytes()); + let secret = Some(signing_key); PartialKeypair { public, secret } } From 0520edaa44afb36e56581c4cb378c9fd90c02c76 Mon Sep 17 00:00:00 2001 From: Timo Tiuraniemi Date: Fri, 18 Aug 2023 13:36:29 +0300 Subject: [PATCH 138/157] Add make_read_only to turn a writeable hypercore into a read-only --- Cargo.toml | 2 +- src/core.rs | 41 ++++++++++++++++++++++---- src/oplog/mod.rs | 70 ++++++++++++++++++++++++++++++++++++--------- tests/common/mod.rs | 39 ++++++++++++++++++++++++- tests/core.rs | 39 ++++++++++++++++++++++++- tests/js_interop.rs | 22 ++------------ 6 files changed, 171 insertions(+), 42 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 0bcce643..82a9d6ba 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "hypercore" -version = "0.12.0-alpha.5" +version = "0.12.0-alpha.6" license = "MIT OR Apache-2.0" description = "Secure, distributed, append-only log" documentation = "https://docs.rs/hypercore" diff --git a/src/core.rs b/src/core.rs index 37cc0244..5ee2a897 100644 --- a/src/core.rs +++ b/src/core.rs @@ -216,6 +216,7 @@ where } }; changeset.ancestors = tree_upgrade.ancestors; + changeset.hash = Some(changeset.hash()); changeset.signature = Some(Signature::try_from(&*tree_upgrade.signature).map_err(|_| { HypercoreError::InvalidSignature { @@ -223,6 +224,14 @@ where } })?); + // Update the header with this changeset to make in-memory value match that + // of the stored value. + oplog_open_outcome.oplog.update_header_with_changeset( + &changeset, + None, + &mut oplog_open_outcome.header, + )?; + // TODO: Skip reorg hints for now, seems to only have to do with replication // addReorgHint(header.hints.reorgs, tree, batch) @@ -317,7 +326,7 @@ where // Now ready to flush if self.should_flush_bitfield_and_tree_and_oplog() { - self.flush_bitfield_and_tree_and_oplog().await?; + self.flush_bitfield_and_tree_and_oplog(false).await?; } } @@ -416,7 +425,7 @@ where // Now ready to flush if self.should_flush_bitfield_and_tree_and_oplog() { - self.flush_bitfield_and_tree_and_oplog().await?; + self.flush_bitfield_and_tree_and_oplog(false).await?; } Ok(()) @@ -533,7 +542,7 @@ where // Now ready to flush if self.should_flush_bitfield_and_tree_and_oplog() { - self.flush_bitfield_and_tree_and_oplog().await?; + self.flush_bitfield_and_tree_and_oplog(false).await?; } Ok(true) } @@ -562,6 +571,23 @@ where } } + /// Makes the hypercore read-only by deleting the secret key. Returns true if the + /// hypercore was changed, false if the hypercore was already read-only. This is useful + /// in scenarios where a hypercore should be made immutable after initial values have + /// been stored. + #[instrument(err, skip_all)] + pub async fn make_read_only(&mut self) -> Result { + if self.key_pair.secret.is_some() { + self.key_pair.secret = None; + self.header.signer.secret = None; + // Need to flush clearing traces to make sure both oplog slots are cleared + self.flush_bitfield_and_tree_and_oplog(true).await?; + Ok(true) + } else { + Ok(false) + } + } + async fn byte_range( &mut self, index: u64, @@ -658,13 +684,16 @@ where } } - async fn flush_bitfield_and_tree_and_oplog(&mut self) -> Result<(), HypercoreError> { + async fn flush_bitfield_and_tree_and_oplog( + &mut self, + clear_traces: bool, + ) -> Result<(), HypercoreError> { let infos = self.bitfield.flush(); self.storage.flush_infos(&infos).await?; let infos = self.tree.flush(); self.storage.flush_infos(&infos).await?; - let infos_to_flush = self.oplog.flush(&self.header)?; - self.storage.flush_infos(&infos_to_flush).await?; + let infos = self.oplog.flush(&self.header, clear_traces)?; + self.storage.flush_infos(&infos).await?; Ok(()) } } diff --git a/src/oplog/mod.rs b/src/oplog/mod.rs index 7cb091f8..ac1651d2 100644 --- a/src/oplog/mod.rs +++ b/src/oplog/mod.rs @@ -13,6 +13,7 @@ pub(crate) use entry::{Entry, EntryTreeUpgrade}; pub(crate) use header::{Header, HeaderTree}; pub(crate) const MAX_OPLOG_ENTRIES_BYTE_SIZE: u64 = 65536; +const HEADER_SIZE: usize = 4096; /// Oplog. /// @@ -64,10 +65,11 @@ impl OplogOpenOutcome { } } +#[repr(usize)] enum OplogSlot { FirstHeader = 0, - SecondHeader = 4096, - Entries = 4096 * 2, + SecondHeader = HEADER_SIZE, + Entries = HEADER_SIZE * 2, } #[derive(Debug)] @@ -179,8 +181,22 @@ impl Oplog { atomic: bool, header: &Header, ) -> Result { - let tree_nodes: Vec = changeset.nodes.clone(); let mut header: Header = header.clone(); + let entry = self.update_header_with_changeset(changeset, bitfield_update, &mut header)?; + + Ok(OplogCreateHeaderOutcome { + header, + infos_to_flush: self.append_entries(&[entry], atomic)?, + }) + } + + pub(crate) fn update_header_with_changeset( + &mut self, + changeset: &MerkleTreeChangeset, + bitfield_update: Option, + header: &mut Header, + ) -> Result { + let tree_nodes: Vec = changeset.nodes.clone(); let entry: Entry = if changeset.upgraded { let hash = changeset .hash @@ -213,11 +229,7 @@ impl Oplog { bitfield: bitfield_update, } }; - - Ok(OplogCreateHeaderOutcome { - header, - infos_to_flush: self.append_entries(&[entry], atomic)?, - }) + Ok(entry) } /// Clears a segment, returns infos to write to storage. @@ -240,8 +252,30 @@ impl Oplog { } /// Flushes pending changes, returns infos to write to storage. - pub(crate) fn flush(&mut self, header: &Header) -> Result, HypercoreError> { - let (new_header_bits, infos_to_flush) = Self::insert_header(header, 0, self.header_bits)?; + pub(crate) fn flush( + &mut self, + header: &Header, + clear_traces: bool, + ) -> Result, HypercoreError> { + let (new_header_bits, infos_to_flush) = if clear_traces { + // When clearing traces, both slots need to be cleared, hence + // do this twice, but for the first time, ignore the truncate + // store info, to end up with three StoreInfos. + let (new_header_bits, infos_to_flush) = + Self::insert_header(header, 0, self.header_bits, clear_traces)?; + let mut combined_infos_to_flush = vec![infos_to_flush + .into_vec() + .drain(0..1) + .into_iter() + .next() + .unwrap()]; + let (new_header_bits, infos_to_flush) = + Self::insert_header(header, 0, new_header_bits, clear_traces)?; + combined_infos_to_flush.extend(infos_to_flush.into_vec()); + (new_header_bits, combined_infos_to_flush.into_boxed_slice()) + } else { + Self::insert_header(header, 0, self.header_bits, clear_traces)? + }; self.entries_byte_length = 0; self.entries_length = 0; self.header_bits = new_header_bits; @@ -290,7 +324,7 @@ impl Oplog { let entries_byte_length: u64 = 0; let header = Header::new(key_pair); let (header_bits, infos_to_flush) = - Self::insert_header(&header, entries_byte_length, INITIAL_HEADER_BITS)?; + Self::insert_header(&header, entries_byte_length, INITIAL_HEADER_BITS, false)?; let oplog = Oplog { header_bits, entries_length, @@ -309,6 +343,7 @@ impl Oplog { header: &Header, entries_byte_length: u64, current_header_bits: [bool; 2], + clear_traces: bool, ) -> Result<([bool; 2], Box<[StoreInfo]>), HypercoreError> { // The first 8 bytes will be filled with `prepend_leader`. let data_start_index: usize = 8; @@ -329,6 +364,15 @@ impl Oplog { // Preencode the new header (*state).preencode(header)?; + // If clearing, lets add zeros to the end + let end = if clear_traces { + let end = state.end(); + state.set_end(HEADER_SIZE); + end + } else { + state.end() + }; + // Create a buffer for the needed data let mut buffer = state.create_buffer(); @@ -337,7 +381,7 @@ impl Oplog { // Finally prepend the buffer's 8 first bytes with a CRC, len and right bits Self::prepend_leader( - state.end() - data_start_index, + end - data_start_index, header_bit, false, &mut state, @@ -345,7 +389,7 @@ impl Oplog { )?; // The oplog is always truncated to the minimum byte size, which is right after - // the all of the entries in the oplog finish. + // all of the entries in the oplog finish. let truncate_index = OplogSlot::Entries as u64 + entries_byte_length; Ok(( new_header_bits, diff --git a/tests/common/mod.rs b/tests/common/mod.rs index 371fa417..1814262f 100644 --- a/tests/common/mod.rs +++ b/tests/common/mod.rs @@ -1,7 +1,11 @@ +use anyhow::Result; use ed25519_dalek::{SigningKey, VerifyingKey, PUBLIC_KEY_LENGTH, SECRET_KEY_LENGTH}; +use random_access_disk::RandomAccessDisk; use sha2::{Digest, Sha256}; +use std::io::prelude::*; +use std::path::Path; -use hypercore::PartialKeypair; +use hypercore::{Hypercore, HypercoreBuilder, PartialKeypair, Storage}; const TEST_PUBLIC_KEY_BYTES: [u8; PUBLIC_KEY_LENGTH] = [ 0x97, 0x60, 0x6c, 0xaa, 0xd2, 0xb0, 0x8c, 0x1d, 0x5f, 0xe1, 0x64, 0x2e, 0xee, 0xa5, 0x62, 0xcb, @@ -31,6 +35,22 @@ pub fn get_test_key_pair() -> PartialKeypair { PartialKeypair { public, secret } } +pub async fn create_hypercore(work_dir: &str) -> Result> { + let path = Path::new(work_dir).to_owned(); + let key_pair = get_test_key_pair(); + let storage = Storage::new_disk(&path, true).await?; + Ok(HypercoreBuilder::new(storage) + .key_pair(key_pair) + .build() + .await?) +} + +pub async fn open_hypercore(work_dir: &str) -> Result> { + let path = Path::new(work_dir).to_owned(); + let storage = Storage::new_disk(&path, false).await?; + Ok(HypercoreBuilder::new(storage).open(true).build().await?) +} + pub fn create_hypercore_hash(dir: &str) -> HypercoreHash { let bitfield = hash_file(format!("{}/bitfield", dir)); let data = hash_file(format!("{}/data", dir)); @@ -63,3 +83,20 @@ pub fn hash_file(file: String) -> Option { } } } + +pub fn storage_contains_data(dir: &Path, data: &[u8]) -> bool { + for file_name in ["bitfield", "data", "oplog", "tree"] { + let file_path = dir.join(file_name); + let mut file = std::fs::File::open(file_path).unwrap(); + let mut buffer = Vec::new(); + file.read_to_end(&mut buffer).unwrap(); + if is_sub(&buffer, data) { + return true; + } + } + false +} + +fn is_sub(haystack: &[T], needle: &[T]) -> bool { + haystack.windows(needle.len()).any(|c| c == needle) +} diff --git a/tests/core.rs b/tests/core.rs index 982d8b18..ac956200 100644 --- a/tests/core.rs +++ b/tests/core.rs @@ -1,8 +1,9 @@ pub mod common; use anyhow::Result; -use common::get_test_key_pair; +use common::{create_hypercore, get_test_key_pair, open_hypercore, storage_contains_data}; use hypercore::{HypercoreBuilder, Storage}; +use tempfile::Builder; use test_log::test; #[test(async_std::test)] @@ -35,3 +36,39 @@ async fn hypercore_open_with_key_pair_error() -> Result<()> { .is_err()); Ok(()) } + +#[test(async_std::test)] +async fn hypercore_make_read_only() -> Result<()> { + let dir = Builder::new() + .prefix("hypercore_make_read_only") + .tempdir() + .unwrap(); + let write_key_pair = { + let mut hypercore = create_hypercore(&dir.path().to_string_lossy()).await?; + hypercore.append(b"Hello").await?; + hypercore.append(b"World!").await?; + hypercore.key_pair().clone() + }; + assert!(storage_contains_data( + dir.path(), + &write_key_pair.secret.as_ref().unwrap().to_bytes() + )); + assert!(write_key_pair.secret.is_some()); + let read_key_pair = { + let mut hypercore = open_hypercore(&dir.path().to_string_lossy()).await?; + assert_eq!(&hypercore.get(0).await?.unwrap(), b"Hello"); + assert_eq!(&hypercore.get(1).await?.unwrap(), b"World!"); + assert!(hypercore.make_read_only().await?); + hypercore.key_pair().clone() + }; + assert!(read_key_pair.secret.is_none()); + assert!(!storage_contains_data( + dir.path(), + &write_key_pair.secret.as_ref().unwrap().to_bytes()[16..], + )); + + let mut hypercore = open_hypercore(&dir.path().to_string_lossy()).await?; + assert_eq!(&hypercore.get(0).await?.unwrap(), b"Hello"); + assert_eq!(&hypercore.get(1).await?.unwrap(), b"World!"); + Ok(()) +} diff --git a/tests/js_interop.rs b/tests/js_interop.rs index 4276c616..fb836415 100644 --- a/tests/js_interop.rs +++ b/tests/js_interop.rs @@ -1,12 +1,10 @@ pub mod common; pub mod js; -use std::{path::Path, sync::Once}; +use std::sync::Once; use anyhow::Result; -use common::{create_hypercore_hash, get_test_key_pair}; -use hypercore::{Hypercore, HypercoreBuilder, Storage}; +use common::{create_hypercore, create_hypercore_hash, open_hypercore}; use js::{cleanup, install, js_run_step, prepare_test_set}; -use random_access_disk::RandomAccessDisk; use test_log::test; #[cfg(feature = "async-std")] @@ -139,22 +137,6 @@ async fn step_5_clear_some(work_dir: &str) -> Result<()> { Ok(()) } -async fn create_hypercore(work_dir: &str) -> Result> { - let path = Path::new(work_dir).to_owned(); - let key_pair = get_test_key_pair(); - let storage = Storage::new_disk(&path, true).await?; - Ok(HypercoreBuilder::new(storage) - .key_pair(key_pair) - .build() - .await?) -} - -async fn open_hypercore(work_dir: &str) -> Result> { - let path = Path::new(work_dir).to_owned(); - let storage = Storage::new_disk(&path, false).await?; - Ok(HypercoreBuilder::new(storage).open(true).build().await?) -} - fn step_0_hash() -> common::HypercoreHash { common::HypercoreHash { bitfield: None, From 13a176f14669f4b7b337e50ed87f2f37a46fca3a Mon Sep 17 00:00:00 2001 From: Timo Tiuraniemi Date: Mon, 21 Aug 2023 08:15:07 +0300 Subject: [PATCH 139/157] Simpler getting a vector with just one element --- src/oplog/mod.rs | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/src/oplog/mod.rs b/src/oplog/mod.rs index ac1651d2..6c720201 100644 --- a/src/oplog/mod.rs +++ b/src/oplog/mod.rs @@ -263,12 +263,8 @@ impl Oplog { // store info, to end up with three StoreInfos. let (new_header_bits, infos_to_flush) = Self::insert_header(header, 0, self.header_bits, clear_traces)?; - let mut combined_infos_to_flush = vec![infos_to_flush - .into_vec() - .drain(0..1) - .into_iter() - .next() - .unwrap()]; + let mut combined_infos_to_flush: Vec = + infos_to_flush.into_vec().drain(0..1).into_iter().collect(); let (new_header_bits, infos_to_flush) = Self::insert_header(header, 0, new_header_bits, clear_traces)?; combined_infos_to_flush.extend(infos_to_flush.into_vec()); From c8840287ab6dfebc60cb84f9196ec3f8278d105f Mon Sep 17 00:00:00 2001 From: Timo Tiuraniemi Date: Mon, 2 Oct 2023 09:27:57 +0300 Subject: [PATCH 140/157] Switch to blake2 from unmaintained blake2-rfc --- Cargo.toml | 6 +---- src/crypto/hash.rs | 63 +++++++++++++++++++++++++--------------------- 2 files changed, 35 insertions(+), 34 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 82a9d6ba..42683e2f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -18,11 +18,7 @@ categories = [ edition = "2018" [dependencies] -# TODO: After this issue: -# https://github.com/RustCrypto/hashes/issues/360 -# that blocks public key => discovery key creation is fixed, switching to the blake2 crate -# would probably be wise. -blake2-rfc = "0.2.18" +blake2 = "0.10.6" byteorder = "1.3.4" ed25519-dalek = { version = "2.0.0", features = ["rand_core"] } thiserror = "1" diff --git a/src/crypto/hash.rs b/src/crypto/hash.rs index 5d4fba2c..41d3e72a 100644 --- a/src/crypto/hash.rs +++ b/src/crypto/hash.rs @@ -1,6 +1,7 @@ -pub(crate) use blake2_rfc::blake2b::Blake2bResult; - -use blake2_rfc::blake2b::Blake2b; +use blake2::{ + digest::{generic_array::GenericArray, typenum::U32, FixedOutput}, + Blake2b, Blake2bMac, Digest, +}; use byteorder::{BigEndian, WriteBytesExt}; use compact_encoding::State; use ed25519_dalek::VerifyingKey; @@ -24,6 +25,9 @@ const TREE: [u8; 32] = [ 0x8B, 0x5A, 0xAD, 0x8B, 0x58, 0x81, 0xBF, 0xC0, 0xAD, 0xB5, 0xEF, 0x38, 0xA3, 0x27, 0x5B, 0x9C, ]; +pub(crate) type Blake2bResult = GenericArray; +type Blake2b256 = Blake2b; + /// `BLAKE2b` hash. #[derive(Debug, Clone, PartialEq)] pub(crate) struct Hash { @@ -36,9 +40,9 @@ impl Hash { pub(crate) fn from_leaf(data: &[u8]) -> Self { let size = u64_as_be(data.len() as u64); - let mut hasher = Blake2b::new(32); - hasher.update(&LEAF_TYPE); - hasher.update(&size); + let mut hasher = Blake2b256::new(); + hasher.update(LEAF_TYPE); + hasher.update(size); hasher.update(data); Self { @@ -57,9 +61,9 @@ impl Hash { let size = u64_as_be(node1.length + node2.length); - let mut hasher = Blake2b::new(32); - hasher.update(&PARENT_TYPE); - hasher.update(&size); + let mut hasher = Blake2b256::new(); + hasher.update(PARENT_TYPE); + hasher.update(size); hasher.update(node1.hash()); hasher.update(node2.hash()); @@ -72,10 +76,11 @@ impl Hash { /// network without leaking the key itself. #[allow(dead_code)] pub(crate) fn for_discovery_key(public_key: VerifyingKey) -> Self { - let mut hasher = Blake2b::with_key(32, public_key.as_bytes()); - hasher.update(&HYPERCORE); + let mut hasher = + Blake2bMac::::new_with_salt_and_personal(public_key.as_bytes(), &[], &[]).unwrap(); + blake2::digest::Update::update(&mut hasher, &HYPERCORE); Self { - hash: hasher.finalize(), + hash: hasher.finalize_fixed(), } } @@ -83,14 +88,14 @@ impl Hash { // Called `crypto.tree()` in the JS implementation. #[allow(dead_code)] pub(crate) fn from_roots(roots: &[impl AsRef]) -> Self { - let mut hasher = Blake2b::new(32); - hasher.update(&ROOT_TYPE); + let mut hasher = Blake2b256::new(); + hasher.update(ROOT_TYPE); for node in roots { let node = node.as_ref(); hasher.update(node.hash()); - hasher.update(&u64_as_be(node.index())); - hasher.update(&u64_as_be(node.len())); + hasher.update(u64_as_be(node.index())); + hasher.update(u64_as_be(node.len())); } Self { @@ -100,7 +105,7 @@ impl Hash { /// Returns a byte slice of this `Hash`'s contents. pub(crate) fn as_bytes(&self) -> &[u8] { - self.hash.as_bytes() + self.hash.as_slice() } // NB: The following methods mirror Javascript naming in @@ -114,8 +119,8 @@ impl Hash { .encode_u64(data.len() as u64, &mut size) .expect("Encoding u64 should not fail"); - let mut hasher = Blake2b::new(32); - hasher.update(&LEAF_TYPE); + let mut hasher = Blake2b256::new(); + hasher.update(LEAF_TYPE); hasher.update(&size); hasher.update(data); @@ -137,8 +142,8 @@ impl Hash { .encode_u64(node1.length + node2.length, &mut size) .expect("Encoding u64 should not fail"); - let mut hasher = Blake2b::new(32); - hasher.update(&PARENT_TYPE); + let mut hasher = Blake2b256::new(); + hasher.update(PARENT_TYPE); hasher.update(&size); hasher.update(node1.hash()); hasher.update(node2.hash()); @@ -150,8 +155,8 @@ impl Hash { /// Hash a tree pub(crate) fn tree(roots: &[impl AsRef]) -> Self { - let mut hasher = Blake2b::new(32); - hasher.update(&ROOT_TYPE); + let mut hasher = Blake2b256::new(); + hasher.update(ROOT_TYPE); for node in roots { let node = node.as_ref(); @@ -221,11 +226,11 @@ mod tests { use data_encoding; fn hash_with_extra_byte(data: &[u8], byte: u8) -> Box<[u8]> { - let mut hasher = Blake2b::new(32); + let mut hasher = Blake2b256::new(); hasher.update(data); - hasher.update(&[byte]); + hasher.update([byte]); let hash = hasher.finalize(); - hash.as_bytes().into() + hash.as_slice().into() } fn hex_bytes(hex: &str) -> Vec { @@ -337,10 +342,10 @@ mod tests { // and validates that our arrays match #[test] fn hash_namespace() { - let mut hasher = Blake2b::new(32); - hasher.update(&HYPERCORE); + let mut hasher = Blake2b256::new(); + hasher.update(HYPERCORE); let hash = hasher.finalize(); - let ns = hash.as_bytes(); + let ns = hash.as_slice(); let tree: Box<[u8]> = { hash_with_extra_byte(ns, 0) }; assert_eq!(tree, TREE.into()); } From de62da935813915d4f4c15200ee0c27e393fcd79 Mon Sep 17 00:00:00 2001 From: Timo Tiuraniemi Date: Tue, 3 Oct 2023 10:53:39 +0300 Subject: [PATCH 141/157] Fix compatibility with JS v10 oplog that includes the manifest Implementation of wire protocol changes related to manifest will need to wait for hypercore v11 --- src/core.rs | 16 ++-- src/crypto/hash.rs | 12 ++- src/crypto/manifest.rs | 43 ++++++++++ src/crypto/mod.rs | 6 +- src/encoding.rs | 116 ++++++++++++++++++++++++++ src/oplog/header.rs | 181 +++++++++++++++++++---------------------- tests/js_interop.rs | 10 +-- 7 files changed, 269 insertions(+), 115 deletions(-) create mode 100644 src/crypto/manifest.rs diff --git a/src/core.rs b/src/core.rs index 5ee2a897..7e2f7c7f 100644 --- a/src/core.rs +++ b/src/core.rs @@ -243,7 +243,7 @@ where let oplog = oplog_open_outcome.oplog; let header = oplog_open_outcome.header; - let key_pair = header.signer.clone(); + let key_pair = header.key_pair.clone(); Ok(Hypercore { key_pair, @@ -262,7 +262,7 @@ where Info { length: self.tree.length, byte_length: self.tree.byte_length, - contiguous_length: self.header.contiguous_length, + contiguous_length: self.header.hints.contiguous_length, fork: self.tree.fork, writeable: self.key_pair.secret.is_some(), } @@ -380,8 +380,8 @@ where self.bitfield.set_range(start, end - start, false); // Set contiguous length - if start < self.header.contiguous_length { - self.header.contiguous_length = start; + if start < self.header.hints.contiguous_length { + self.header.hints.contiguous_length = start; } // Find the biggest hole that can be punched into the data @@ -579,7 +579,7 @@ where pub async fn make_read_only(&mut self) -> Result { if self.key_pair.secret.is_some() { self.key_pair.secret = None; - self.header.signer.secret = None; + self.header.key_pair.secret = None; // Need to flush clearing traces to make sure both oplog slots are cleared self.flush_bitfield_and_tree_and_oplog(true).await?; Ok(true) @@ -704,7 +704,7 @@ fn update_contiguous_length( bitfield_update: &BitfieldUpdate, ) { let end = bitfield_update.start + bitfield_update.length; - let mut c = header.contiguous_length; + let mut c = header.hints.contiguous_length; if bitfield_update.drop { if c <= end && c > bitfield_update.start { c = bitfield_update.start; @@ -716,8 +716,8 @@ fn update_contiguous_length( } } - if c != header.contiguous_length { - header.contiguous_length = c; + if c != header.hints.contiguous_length { + header.hints.contiguous_length = c; } } diff --git a/src/crypto/hash.rs b/src/crypto/hash.rs index 41d3e72a..f744048d 100644 --- a/src/crypto/hash.rs +++ b/src/crypto/hash.rs @@ -19,12 +19,22 @@ const ROOT_TYPE: [u8; 1] = [0x02]; const HYPERCORE: [u8; 9] = *b"hypercore"; // These the output of, see `hash_namespace` test below for how they are produced -// https://github.com/hypercore-protocol/hypercore/blob/70b271643c4e4b1e5ecae5bb579966dfe6361ff3/lib/caps.js#L9 +// https://github.com/holepunchto/hypercore/blob/cf08b72f14ed7d9ef6d497ebb3071ee0ae20967e/lib/caps.js#L16 const TREE: [u8; 32] = [ 0x9F, 0xAC, 0x70, 0xB5, 0xC, 0xA1, 0x4E, 0xFC, 0x4E, 0x91, 0xC8, 0x33, 0xB2, 0x4, 0xE7, 0x5B, 0x8B, 0x5A, 0xAD, 0x8B, 0x58, 0x81, 0xBF, 0xC0, 0xAD, 0xB5, 0xEF, 0x38, 0xA3, 0x27, 0x5B, 0x9C, ]; +// const DEFAULT_NAMESPACE: [u8; 32] = [ +// 0x41, 0x44, 0xEE, 0xA5, 0x31, 0xE4, 0x83, 0xD5, 0x4E, 0x0C, 0x14, 0xF4, 0xCA, 0x68, 0xE0, 0x64, +// 0x4F, 0x35, 0x53, 0x43, 0xFF, 0x6F, 0xCB, 0x0F, 0x00, 0x52, 0x00, 0xE1, 0x2C, 0xD7, 0x47, 0xCB, +// ]; + +// const MANIFEST: [u8; 32] = [ +// 0xE6, 0x4B, 0x71, 0x08, 0xEA, 0xCC, 0xE4, 0x7C, 0xFC, 0x61, 0xAC, 0x85, 0x05, 0x68, 0xF5, 0x5F, +// 0x8B, 0x15, 0xB8, 0x2E, 0xC5, 0xED, 0x78, 0xC4, 0xEC, 0x59, 0x7B, 0x03, 0x6E, 0x2A, 0x14, 0x98, +// ]; + pub(crate) type Blake2bResult = GenericArray; type Blake2b256 = Blake2b; diff --git a/src/crypto/manifest.rs b/src/crypto/manifest.rs new file mode 100644 index 00000000..b3900c5a --- /dev/null +++ b/src/crypto/manifest.rs @@ -0,0 +1,43 @@ +// These the output of the following link: +// https://github.com/holepunchto/hypercore/blob/cf08b72f14ed7d9ef6d497ebb3071ee0ae20967e/lib/caps.js#L16 + +const DEFAULT_NAMESPACE: [u8; 32] = [ + 0x41, 0x44, 0xEE, 0xA5, 0x31, 0xE4, 0x83, 0xD5, 0x4E, 0x0C, 0x14, 0xF4, 0xCA, 0x68, 0xE0, 0x64, + 0x4F, 0x35, 0x53, 0x43, 0xFF, 0x6F, 0xCB, 0x0F, 0x00, 0x52, 0x00, 0xE1, 0x2C, 0xD7, 0x47, 0xCB, +]; + +// TODO: Eventually this would be used in manifestHash +// https://github.com/holepunchto/hypercore/blob/cf08b72f14ed7d9ef6d497ebb3071ee0ae20967e/lib/manifest.js#L211 +// +// const MANIFEST: [u8; 32] = [ +// 0xE6, 0x4B, 0x71, 0x08, 0xEA, 0xCC, 0xE4, 0x7C, 0xFC, 0x61, 0xAC, 0x85, 0x05, 0x68, 0xF5, 0x5F, +// 0x8B, 0x15, 0xB8, 0x2E, 0xC5, 0xED, 0x78, 0xC4, 0xEC, 0x59, 0x7B, 0x03, 0x6E, 0x2A, 0x14, 0x98, +// ]; + +#[derive(Debug, Clone)] +pub(crate) struct Manifest { + pub(crate) hash: String, + // TODO: In v11 can be static + // pub(crate) static_core: Option, + pub(crate) signer: ManifestSigner, + // TODO: In v11 can have multiple signers + // pub(crate) multiple_signers: Option, +} + +#[derive(Debug, Clone)] +pub(crate) struct ManifestSigner { + pub(crate) signature: String, + pub(crate) namespace: [u8; 32], + pub(crate) public_key: [u8; 32], +} + +pub(crate) fn default_signer_manifest(public_key: [u8; 32]) -> Manifest { + Manifest { + hash: "blake2b".to_string(), + signer: ManifestSigner { + signature: "ed25519".to_string(), + namespace: DEFAULT_NAMESPACE, + public_key, + }, + } +} diff --git a/src/crypto/mod.rs b/src/crypto/mod.rs index e9eec459..1bf2ab5b 100644 --- a/src/crypto/mod.rs +++ b/src/crypto/mod.rs @@ -2,6 +2,8 @@ mod hash; mod key_pair; +mod manifest; -pub(crate) use self::hash::{signable_tree, Hash}; -pub use self::key_pair::{generate as generate_signing_key, sign, verify, PartialKeypair}; +pub(crate) use hash::{signable_tree, Hash}; +pub use key_pair::{generate as generate_signing_key, sign, verify, PartialKeypair}; +pub(crate) use manifest::{default_signer_manifest, Manifest, ManifestSigner}; diff --git a/src/encoding.rs b/src/encoding.rs index 8f715fd1..ed049a65 100644 --- a/src/encoding.rs +++ b/src/encoding.rs @@ -1,8 +1,10 @@ //! Hypercore-specific compact encodings pub use compact_encoding::{CompactEncoding, EncodingError, EncodingErrorKind, State}; +use std::convert::TryInto; use std::ops::{Deref, DerefMut}; use crate::{ + crypto::{Manifest, ManifestSigner}, DataBlock, DataHash, DataSeek, DataUpgrade, Node, RequestBlock, RequestSeek, RequestUpgrade, }; @@ -252,3 +254,117 @@ impl CompactEncoding for HypercoreState { }) } } + +impl CompactEncoding for State { + fn preencode(&mut self, value: &Manifest) -> Result { + self.add_end(1)?; // Version + self.add_end(1)?; // hash in one byte + self.add_end(1)?; // type in one byte + self.preencode(&value.signer) + } + + fn encode(&mut self, value: &Manifest, buffer: &mut [u8]) -> Result { + self.set_byte_to_buffer(0, buffer)?; // Version + if &value.hash == "blake2b" { + self.set_byte_to_buffer(0, buffer)?; // Version + } else { + return Err(EncodingError::new( + EncodingErrorKind::InvalidData, + &format!("Unknown hash: {}", &value.hash), + )); + } + // Type. 0: static, 1: signer, 2: multiple signers + self.set_byte_to_buffer(1, buffer)?; // Version + self.encode(&value.signer, buffer) + } + + fn decode(&mut self, buffer: &[u8]) -> Result { + let version: u8 = self.decode_u8(buffer)?; + if version != 0 { + panic!("Unknown manifest version {}", version); + } + let hash_id: u8 = self.decode_u8(buffer)?; + let hash: String = if hash_id != 0 { + return Err(EncodingError::new( + EncodingErrorKind::InvalidData, + &format!("Unknown hash id: {hash_id}"), + )); + } else { + "blake2b".to_string() + }; + + let manifest_type: u8 = self.decode_u8(buffer)?; + if manifest_type != 1 { + return Err(EncodingError::new( + EncodingErrorKind::InvalidData, + &format!("Unknown manifest type: {manifest_type}"), + )); + } + let signer: ManifestSigner = self.decode(buffer)?; + + Ok(Manifest { hash, signer }) + } +} + +impl CompactEncoding for State { + fn preencode(&mut self, _value: &ManifestSigner) -> Result { + self.add_end(1)?; // Signature + self.preencode_fixed_32()?; + self.preencode_fixed_32() + } + + fn encode( + &mut self, + value: &ManifestSigner, + buffer: &mut [u8], + ) -> Result { + if &value.signature == "ed25519" { + self.set_byte_to_buffer(0, buffer)?; + } else { + return Err(EncodingError::new( + EncodingErrorKind::InvalidData, + &format!("Unknown signature type: {}", &value.signature), + )); + } + self.encode_fixed_32(&value.namespace, buffer)?; + self.encode_fixed_32(&value.public_key, buffer) + } + + fn decode(&mut self, buffer: &[u8]) -> Result { + let signature_id: u8 = self.decode_u8(buffer)?; + let signature: String = if signature_id != 0 { + return Err(EncodingError::new( + EncodingErrorKind::InvalidData, + &format!("Unknown signature id: {signature_id}"), + )); + } else { + "ed25519".to_string() + }; + let namespace: [u8; 32] = + self.decode_fixed_32(buffer)? + .to_vec() + .try_into() + .map_err(|_err| { + EncodingError::new( + EncodingErrorKind::InvalidData, + "Invalid namespace in manifest signer", + ) + })?; + let public_key: [u8; 32] = + self.decode_fixed_32(buffer)? + .to_vec() + .try_into() + .map_err(|_err| { + EncodingError::new( + EncodingErrorKind::InvalidData, + "Invalid public key in manifest signer", + ) + })?; + + Ok(ManifestSigner { + signature, + namespace, + public_key, + }) + } +} diff --git a/src/oplog/header.rs b/src/oplog/header.rs index 6596e00b..aa27dcec 100644 --- a/src/oplog/header.rs +++ b/src/oplog/header.rs @@ -1,92 +1,63 @@ +use compact_encoding::EncodingErrorKind; use compact_encoding::{CompactEncoding, EncodingError, State}; use ed25519_dalek::{SigningKey, PUBLIC_KEY_LENGTH, SECRET_KEY_LENGTH}; use std::convert::TryInto; +use crate::crypto::default_signer_manifest; +use crate::crypto::Manifest; use crate::PartialKeypair; use crate::VerifyingKey; /// Oplog header. #[derive(Debug, Clone)] pub(crate) struct Header { - pub(crate) types: HeaderTypes, + // TODO: v11 has external + // pub(crate) external: Option, + // NB: This is the manifest hash in v11, right now + // just the public key, + pub(crate) key: [u8; 32], + pub(crate) manifest: Manifest, + pub(crate) key_pair: PartialKeypair, // TODO: This is a keyValueArray in JS pub(crate) user_data: Vec, pub(crate) tree: HeaderTree, - pub(crate) signer: PartialKeypair, pub(crate) hints: HeaderHints, - pub(crate) contiguous_length: u64, } impl Header { /// Creates a new Header from given key pair - pub(crate) fn new(signing_key: PartialKeypair) -> Self { + pub(crate) fn new(key_pair: PartialKeypair) -> Self { + let key = key_pair.public.to_bytes(); + let manifest = default_signer_manifest(key); Self { - types: HeaderTypes::new(), + key, + manifest, + key_pair, user_data: vec![], tree: HeaderTree::new(), - signer: signing_key, - hints: HeaderHints { reorgs: vec![] }, - contiguous_length: 0, + hints: HeaderHints { + reorgs: vec![], + contiguous_length: 0, + }, } // Javascript side, initial header - // // header = { - // types: { tree: 'blake2b', bitfield: 'raw', signer: 'ed25519' }, - // userData: [], - // tree: { - // fork: 0, - // length: 0, - // rootHash: null, - // signature: null - // }, - // signer: opts.keyPair || crypto.keyPair(), - // hints: { - // reorgs: [] - // }, - // contiguousLength: 0 - // } - } -} - -/// Oplog header types -#[derive(Debug, PartialEq, Clone)] -pub(crate) struct HeaderTypes { - pub(crate) tree: String, - pub(crate) bitfield: String, - pub(crate) signer: String, -} -impl HeaderTypes { - pub(crate) fn new() -> Self { - Self { - tree: "blake2b".to_string(), - bitfield: "raw".to_string(), - signer: "ed25519".to_string(), - } - } -} - -impl CompactEncoding for State { - fn preencode(&mut self, value: &HeaderTypes) -> Result { - self.preencode(&value.tree)?; - self.preencode(&value.bitfield)?; - self.preencode(&value.signer) - } - - fn encode(&mut self, value: &HeaderTypes, buffer: &mut [u8]) -> Result { - self.encode(&value.tree, buffer)?; - self.encode(&value.bitfield, buffer)?; - self.encode(&value.signer, buffer) - } - - fn decode(&mut self, buffer: &[u8]) -> Result { - let tree: String = self.decode(buffer)?; - let bitfield: String = self.decode(buffer)?; - let signer: String = self.decode(buffer)?; - Ok(HeaderTypes { - tree, - bitfield, - signer, - }) + // external: null, + // key: opts.key || (compat ? manifest.signer.publicKey : manifestHash(manifest)), + // manifest, + // keyPair, + // userData: [], + // tree: { + // fork: 0, + // length: 0, + // rootHash: null, + // signature: null + // }, + // hints: { + // reorgs: [], + // contiguousLength: 0 + // } + // } } } @@ -197,20 +168,24 @@ impl CompactEncoding for State { #[derive(Debug, Clone)] pub(crate) struct HeaderHints { pub(crate) reorgs: Vec, + pub(crate) contiguous_length: u64, } impl CompactEncoding for State { fn preencode(&mut self, value: &HeaderHints) -> Result { - self.preencode(&value.reorgs) + self.preencode(&value.reorgs)?; + self.preencode(&value.contiguous_length) } fn encode(&mut self, value: &HeaderHints, buffer: &mut [u8]) -> Result { - self.encode(&value.reorgs, buffer) + self.encode(&value.reorgs, buffer)?; + self.encode(&value.contiguous_length, buffer) } fn decode(&mut self, buffer: &[u8]) -> Result { Ok(HeaderHints { reorgs: self.decode(buffer)?, + contiguous_length: self.decode(buffer)?, }) } } @@ -218,43 +193,56 @@ impl CompactEncoding for State { impl CompactEncoding
for State { fn preencode(&mut self, value: &Header) -> Result { self.add_end(1)?; // Version - self.preencode(&value.types)?; + self.add_end(1)?; // Flags + self.preencode_fixed_32()?; // key + self.preencode(&value.manifest)?; + self.preencode(&value.key_pair)?; self.preencode(&value.user_data)?; self.preencode(&value.tree)?; - self.preencode(&value.signer)?; - self.preencode(&value.hints)?; - self.preencode(&value.contiguous_length) + self.preencode(&value.hints) } fn encode(&mut self, value: &Header, buffer: &mut [u8]) -> Result { - self.set_byte_to_buffer(0, buffer)?; // Version - self.encode(&value.types, buffer)?; + self.set_byte_to_buffer(1, buffer)?; // Version + let flags: u8 = 2 | 4; // Manifest and key pair, TODO: external=1 + self.set_byte_to_buffer(flags, buffer)?; + self.encode_fixed_32(&value.key, buffer)?; + self.encode(&value.manifest, buffer)?; + self.encode(&value.key_pair, buffer)?; self.encode(&value.user_data, buffer)?; self.encode(&value.tree, buffer)?; - self.encode(&value.signer, buffer)?; - self.encode(&value.hints, buffer)?; - self.encode(&value.contiguous_length, buffer) + self.encode(&value.hints, buffer) } fn decode(&mut self, buffer: &[u8]) -> Result { let version: u8 = self.decode_u8(buffer)?; - if version != 0 { + if version != 1 { panic!("Unknown oplog version {}", version); } - let types: HeaderTypes = self.decode(buffer)?; + let _flags: u8 = self.decode_u8(buffer)?; + let key: [u8; 32] = self + .decode_fixed_32(buffer)? + .to_vec() + .try_into() + .map_err(|_err| { + EncodingError::new( + EncodingErrorKind::InvalidData, + "Invalid key in oplog header", + ) + })?; + let manifest: Manifest = self.decode(buffer)?; + let key_pair: PartialKeypair = self.decode(buffer)?; let user_data: Vec = self.decode(buffer)?; let tree: HeaderTree = self.decode(buffer)?; - let signer: PartialKeypair = self.decode(buffer)?; let hints: HeaderHints = self.decode(buffer)?; - let contiguous_length: u64 = self.decode(buffer)?; Ok(Header { - types, + key, + manifest, + key_pair, user_data, tree, - signer, hints, - contiguous_length, }) } } @@ -265,20 +253,6 @@ mod tests { use crate::crypto::generate_signing_key; - #[test] - fn encode_header_types() -> Result<(), EncodingError> { - let mut enc_state = State::new_with_start_and_end(8, 8); - let header_types = HeaderTypes::new(); - enc_state.preencode(&header_types)?; - let mut buffer = enc_state.create_buffer(); - enc_state.encode(&header_types, &mut buffer)?; - let mut dec_state = State::from_buffer(&buffer); - dec_state.add_start(8)?; - let header_types_ret: HeaderTypes = dec_state.decode(&buffer)?; - assert_eq!(header_types, header_types_ret); - Ok(()) - } - #[test] fn encode_partial_key_pair() -> Result<(), EncodingError> { let mut enc_state = State::new(); @@ -333,10 +307,19 @@ mod tests { enc_state.encode(&header, &mut buffer)?; let mut dec_state = State::from_buffer(&buffer); let header_ret: Header = dec_state.decode(&buffer)?; - assert_eq!(header.signer.public, header_ret.signer.public); + assert_eq!(header.key_pair.public, header_ret.key_pair.public); assert_eq!(header.tree.fork, header_ret.tree.fork); assert_eq!(header.tree.length, header_ret.tree.length); - assert_eq!(header.types, header_ret.types); + assert_eq!(header.tree.length, header_ret.tree.length); + assert_eq!(header.manifest.hash, header_ret.manifest.hash); + assert_eq!( + header.manifest.signer.public_key, + header_ret.manifest.signer.public_key + ); + assert_eq!( + header.manifest.signer.signature, + header_ret.manifest.signer.signature + ); Ok(()) } } diff --git a/tests/js_interop.rs b/tests/js_interop.rs index fb836415..5d02d737 100644 --- a/tests/js_interop.rs +++ b/tests/js_interop.rs @@ -150,7 +150,7 @@ fn step_1_hash() -> common::HypercoreHash { common::HypercoreHash { bitfield: None, data: None, - oplog: Some("C5C042D47C25465FA708BB0384C88485E1C1AF848FC5D9E6DE34FAF1E88E41A9".into()), + oplog: Some("A30BD5326139E8650F3D53CB43291945AE92796ABAEBE1365AC1B0C37D008936".into()), tree: None, } } @@ -159,7 +159,7 @@ fn step_2_hash() -> common::HypercoreHash { common::HypercoreHash { bitfield: Some("0E2E1FF956A39192CBB68D2212288FE75B32733AB0C442B9F0471E254A0382A2".into()), data: Some("872E4E50CE9990D8B041330C47C9DDD11BEC6B503AE9386A99DA8584E9BB12C4".into()), - oplog: Some("E374F3CFEA62D333E3ADE22A24A0EA50E5AF09CF45E2DEDC0F56F5A214081156".into()), + oplog: Some("C65A6867991D29FCF98B4E4549C1039CB5B3C63D891BA1EA4F0BB47211BA4B05".into()), tree: Some("8577B24ADC763F65D562CD11204F938229AD47F27915B0821C46A0470B80813A".into()), } } @@ -168,7 +168,7 @@ fn step_3_hash() -> common::HypercoreHash { common::HypercoreHash { bitfield: Some("DEC1593A7456C8C9407B9B8B9C89682DFFF33C3892BCC9D9F06956FEE0A1B949".into()), data: Some("99EB5BC150A1102A7E50D15F90594660010B7FE719D54129065D1D417AA5015A".into()), - oplog: Some("6CC9AB30A4146955886937FB442B86716C1E3C3517772EADA62ACF22D4922EAE".into()), + oplog: Some("5DCE3C7C86B0E129B32E5A07CA3DF668006A42F9D75399D6E4DB3F18256B8468".into()), tree: Some("38788609A8634DC8D34F9AE723F3169ADB20768ACFDFF266A43B7E217750DD1E".into()), } } @@ -177,7 +177,7 @@ fn step_4_hash() -> common::HypercoreHash { common::HypercoreHash { bitfield: Some("9B844E9378A7D13D6CDD4C1FF12FB313013E5CC472C6CB46497033563FE6B8F1".into()), data: Some("AF3AC31CFBE1733C62496CF8E856D5F1EFB4B06CBF1E74204221C89E2F3E1CDE".into()), - oplog: Some("DD00EFAE87A5272281EB040ACCF387D0401895B32DEBE4E9313FCFD0CA2B76AE".into()), + oplog: Some("46E01E9CECDF6E7EA85807F65C5F3CEED96583F3BF97BC6835A6DA05E39FE8E9".into()), tree: Some("26339A21D606A1F731B90E8001030651D48378116B06A9C1EF87E2538194C2C6".into()), } } @@ -186,7 +186,7 @@ fn step_5_hash() -> common::HypercoreHash { common::HypercoreHash { bitfield: Some("40C9CED82AE0B7A397C9FDD14EEB7F70B74E8F1229F3ED931852591972DDC3E0".into()), data: Some("D9FFCCEEE9109751F034ECDAE328672956B90A6E0B409C3173741B8A5D0E75AB".into()), - oplog: Some("AC0E5339F3DC58D7875A60B19D4B3AC7BA34990356CF7B8C1A0BC66FF8F31EFB".into()), + oplog: Some("803384F10871FB60E53A7F833E6E1E9729C6D040D960164077963092BBEBA274".into()), tree: Some("26339A21D606A1F731B90E8001030651D48378116B06A9C1EF87E2538194C2C6".into()), } } From 01795b809d8556701528bd074389944ea7862d6d Mon Sep 17 00:00:00 2001 From: Timo Tiuraniemi Date: Tue, 3 Oct 2023 11:07:31 +0300 Subject: [PATCH 142/157] Bump version --- Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index 42683e2f..7fbe133a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "hypercore" -version = "0.12.0-alpha.6" +version = "0.12.0-alpha.7" license = "MIT OR Apache-2.0" description = "Secure, distributed, append-only log" documentation = "https://docs.rs/hypercore" From 1b99951d872b6e801db325736723c61aee02298e Mon Sep 17 00:00:00 2001 From: Timo Tiuraniemi Date: Tue, 3 Oct 2023 13:33:24 +0300 Subject: [PATCH 143/157] Fix clippy warnings --- src/common/error.rs | 6 +++--- src/core.rs | 2 +- src/storage/mod.rs | 6 ++---- src/tree/merkle_tree.rs | 11 +++++------ 4 files changed, 11 insertions(+), 14 deletions(-) diff --git a/src/common/error.rs b/src/common/error.rs index 4a55209d..89ec0b37 100644 --- a/src/common/error.rs +++ b/src/common/error.rs @@ -35,7 +35,7 @@ pub enum HypercoreError { }, /// Corrupt storage #[error("Corrupt storage: {store}.{}", - .context.as_ref().map_or_else(String::new, |ctx| format!(" Context: {}.", ctx)))] + .context.as_ref().map_or_else(String::new, |ctx| format!(" Context: {ctx}.")))] CorruptStorage { /// Store that was corrupt store: Store, @@ -50,7 +50,7 @@ pub enum HypercoreError { }, /// Unexpected IO error occured #[error("Unrecoverable input/output error occured.{}", - .context.as_ref().map_or_else(String::new, |ctx| format!(" {}.", ctx)))] + .context.as_ref().map_or_else(String::new, |ctx| format!(" {ctx}.")))] IO { /// Context for the error context: Option, @@ -72,7 +72,7 @@ impl From for HypercoreError { impl From for HypercoreError { fn from(err: EncodingError) -> Self { Self::InvalidOperation { - context: format!("Encoding failed: {}", err), + context: format!("Encoding failed: {err}"), } } } diff --git a/src/core.rs b/src/core.rs index 7e2f7c7f..427b253a 100644 --- a/src/core.rs +++ b/src/core.rs @@ -407,7 +407,7 @@ where Either::Right(value) => value, Either::Left(_) => { return Err(HypercoreError::InvalidOperation { - context: format!("Could not read offset for index {} from tree", start), + context: format!("Could not read offset for index {start} from tree"), }); } } diff --git a/src/storage/mod.rs b/src/storage/mod.rs index b8745438..c989afe0 100644 --- a/src/storage/mod.rs +++ b/src/storage/mod.rs @@ -34,8 +34,7 @@ pub(crate) fn map_random_access_err(err: RandomAccessError) -> HypercoreError { source, } => HypercoreError::IO { context: Some(format!( - "RandomAccess IO error. Context: {:?}, return_code: {:?}", - context, return_code + "RandomAccess IO error. Context: {context:?}, return_code: {return_code:?}", )), source, }, @@ -45,8 +44,7 @@ pub(crate) fn map_random_access_err(err: RandomAccessError) -> HypercoreError { length, } => HypercoreError::InvalidOperation { context: format!( - "RandomAccess out of bounds. Offset: {}, end: {:?}, length: {}", - offset, end, length + "RandomAccess out of bounds. Offset: {offset}, end: {end:?}, length: {length}", ), }, } diff --git a/src/tree/merkle_tree.rs b/src/tree/merkle_tree.rs index 3bb9359d..684eed61 100644 --- a/src/tree/merkle_tree.rs +++ b/src/tree/merkle_tree.rs @@ -688,7 +688,7 @@ impl MerkleTree { }; if compare_index >= head { return Err(HypercoreError::BadArgument { - context: format!("Hypercore index {} is out of bounds", hypercore_index), + context: format!("Hypercore index {hypercore_index} is out of bounds"), }); } Ok(index) @@ -760,7 +760,7 @@ impl MerkleTree { } Err(HypercoreError::BadArgument { - context: format!("Could not calculate byte offset for index {}", index), + context: format!("Could not calculate byte offset for index {index}"), }) } @@ -1548,8 +1548,7 @@ fn nodes_to_root(index: u64, nodes: u64, head: u64) -> Result= self.nodes.len() { return Err(HypercoreError::InvalidOperation { - context: format!("Expected node {}, got (nil)", index), + context: format!("Expected node {index}, got (nil)"), }); } let node = self.nodes[self.i].clone(); self.i += 1; if node.index != index { return Err(HypercoreError::InvalidOperation { - context: format!("Expected node {}, got node {}", index, node.index), + context: format!("Expected node {index}, got node {}", node.index), }); } self.length -= 1; From 16e63c3e762ddde63060d955682132a15972659e Mon Sep 17 00:00:00 2001 From: Timo Tiuraniemi Date: Tue, 3 Oct 2023 13:42:21 +0300 Subject: [PATCH 144/157] Fix tests for the tokio runtime --- tests/core.rs | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/tests/core.rs b/tests/core.rs index ac956200..f3e8d2ec 100644 --- a/tests/core.rs +++ b/tests/core.rs @@ -6,14 +6,19 @@ use hypercore::{HypercoreBuilder, Storage}; use tempfile::Builder; use test_log::test; -#[test(async_std::test)] +#[cfg(feature = "async-std")] +use async_std::test as async_test; +#[cfg(feature = "tokio")] +use tokio::test as async_test; + +#[test(async_test)] async fn hypercore_new() -> Result<()> { let storage = Storage::new_memory().await?; let _hypercore = HypercoreBuilder::new(storage).build(); Ok(()) } -#[test(async_std::test)] +#[test(async_test)] async fn hypercore_new_with_key_pair() -> Result<()> { let storage = Storage::new_memory().await?; let key_pair = get_test_key_pair(); @@ -24,7 +29,7 @@ async fn hypercore_new_with_key_pair() -> Result<()> { Ok(()) } -#[test(async_std::test)] +#[test(async_test)] async fn hypercore_open_with_key_pair_error() -> Result<()> { let storage = Storage::new_memory().await?; let key_pair = get_test_key_pair(); @@ -37,7 +42,7 @@ async fn hypercore_open_with_key_pair_error() -> Result<()> { Ok(()) } -#[test(async_std::test)] +#[test(async_test)] async fn hypercore_make_read_only() -> Result<()> { let dir = Builder::new() .prefix("hypercore_make_read_only") From 7269a0f89f40508c53ae9769d9717e6c817e19aa Mon Sep 17 00:00:00 2001 From: Timo Tiuraniemi Date: Tue, 3 Oct 2023 14:55:01 +0300 Subject: [PATCH 145/157] Fix benches to work also with tokio --- Cargo.toml | 2 +- benches/disk.rs | 138 +++++++++++++++++++++++++++++----------------- benches/memory.rs | 130 ++++++++++++++++++++++++++----------------- 3 files changed, 168 insertions(+), 102 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 7fbe133a..3e843649 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -55,7 +55,7 @@ tempfile = "3.1.0" async-std = { version = "1.12.0", features = ["attributes"] } tokio = { version = "1.27.0", default-features = false, features = ["macros", "rt", "rt-multi-thread"] } sha2 = "0.10.2" -criterion = { version = "0.4", features = ["async_std"] } +criterion = { version = "0.4", features = ["async_std", "async_tokio"] } test-log = { version = "0.2.11", default-features = false, features = ["trace"] } tracing-subscriber = { version = "0.3.16", features = ["env-filter", "fmt"] } diff --git a/benches/disk.rs b/benches/disk.rs index fe5b46ac..326f57b3 100644 --- a/benches/disk.rs +++ b/benches/disk.rs @@ -1,11 +1,28 @@ use std::time::{Duration, Instant}; +#[cfg(feature = "async-std")] use criterion::async_executor::AsyncStdExecutor; use criterion::{black_box, criterion_group, criterion_main, Criterion}; use hypercore::{Hypercore, HypercoreBuilder, HypercoreError, Storage}; use random_access_disk::RandomAccessDisk; use tempfile::Builder as TempfileBuilder; +fn bench_create_disk(c: &mut Criterion) { + let mut group = c.benchmark_group("slow_call"); + group.measurement_time(Duration::from_secs(20)); + + #[cfg(feature = "async-std")] + group.bench_function("create_disk", move |b| { + b.to_async(AsyncStdExecutor) + .iter(|| create_hypercore("create")); + }); + #[cfg(feature = "tokio")] + group.bench_function("create_disk", move |b| { + let rt = tokio::runtime::Runtime::new().unwrap(); + b.to_async(&rt).iter(|| create_hypercore("create")); + }); +} + #[cfg(feature = "cache")] async fn create_hypercore(name: &str) -> Result, HypercoreError> { let dir = TempfileBuilder::new() @@ -31,72 +48,93 @@ async fn create_hypercore(name: &str) -> Result, Hyp HypercoreBuilder::new(storage).build().await } -fn create_disk(c: &mut Criterion) { +fn bench_write_disk(c: &mut Criterion) { let mut group = c.benchmark_group("slow_call"); group.measurement_time(Duration::from_secs(20)); - group.bench_function("create_disk", move |b| { - b.to_async(AsyncStdExecutor) - .iter(|| create_hypercore("create")); + + #[cfg(feature = "async-std")] + group.bench_function("write disk", |b| { + b.to_async(AsyncStdExecutor).iter_custom(write_disk); }); + #[cfg(feature = "tokio")] + group.bench_function("write disk", |b| { + let rt = tokio::runtime::Runtime::new().unwrap(); + b.to_async(&rt).iter_custom(write_disk); + }); +} + +async fn write_disk(iters: u64) -> Duration { + let mut hypercore = create_hypercore("write").await.unwrap(); + let data = Vec::from("hello"); + let start = Instant::now(); + for _ in 0..iters { + black_box(hypercore.append(&data).await.unwrap()); + } + start.elapsed() } -fn write_disk(c: &mut Criterion) { +fn bench_read_disk(c: &mut Criterion) { let mut group = c.benchmark_group("slow_call"); group.measurement_time(Duration::from_secs(20)); - group.bench_function("write_disk", move |b| { - b.to_async(AsyncStdExecutor) - .iter_custom(|iters| async move { - let mut hypercore = create_hypercore("write").await.unwrap(); - let data = Vec::from("hello"); - let start = Instant::now(); - for _ in 0..iters { - black_box(hypercore.append(&data).await.unwrap()); - } - start.elapsed() - }); + + #[cfg(feature = "async-std")] + group.bench_function("read disk", |b| { + b.to_async(AsyncStdExecutor).iter_custom(read_disk); + }); + #[cfg(feature = "tokio")] + group.bench_function("read disk", |b| { + let rt = tokio::runtime::Runtime::new().unwrap(); + b.to_async(&rt).iter_custom(read_disk); }); } -fn read_disk(c: &mut Criterion) { +async fn read_disk(iters: u64) -> Duration { + let mut hypercore = create_hypercore("read").await.unwrap(); + let data = Vec::from("hello"); + for _ in 0..iters { + hypercore.append(&data).await.unwrap(); + } + let start = Instant::now(); + for i in 0..iters { + black_box(hypercore.get(i).await.unwrap()); + } + start.elapsed() +} + +fn bench_clear_disk(c: &mut Criterion) { let mut group = c.benchmark_group("slow_call"); group.measurement_time(Duration::from_secs(20)); - group.bench_function("read_disk", move |b| { - b.to_async(AsyncStdExecutor) - .iter_custom(|iters| async move { - let mut hypercore = create_hypercore("read").await.unwrap(); - let data = Vec::from("hello"); - for _ in 0..iters { - hypercore.append(&data).await.unwrap(); - } - let start = Instant::now(); - for i in 0..iters { - black_box(hypercore.get(i).await.unwrap()); - } - start.elapsed() - }); + + #[cfg(feature = "async-std")] + group.bench_function("clear disk", |b| { + b.to_async(AsyncStdExecutor).iter_custom(clear_disk); + }); + #[cfg(feature = "tokio")] + group.bench_function("clear disk", |b| { + let rt = tokio::runtime::Runtime::new().unwrap(); + b.to_async(&rt).iter_custom(clear_disk); }); } #[allow(clippy::unit_arg)] -fn clear_disk(c: &mut Criterion) { - let mut group = c.benchmark_group("slow_call"); - group.measurement_time(Duration::from_secs(20)); - group.bench_function("clear_disk", move |b| { - b.to_async(AsyncStdExecutor) - .iter_custom(|iters| async move { - let mut hypercore = create_hypercore("clear").await.unwrap(); - let data = Vec::from("hello"); - for _ in 0..iters { - hypercore.append(&data).await.unwrap(); - } - let start = Instant::now(); - for i in 0..iters { - black_box(hypercore.clear(i, 1).await.unwrap()); - } - start.elapsed() - }); - }); +async fn clear_disk(iters: u64) -> Duration { + let mut hypercore = create_hypercore("clear").await.unwrap(); + let data = Vec::from("hello"); + for _ in 0..iters { + hypercore.append(&data).await.unwrap(); + } + let start = Instant::now(); + for i in 0..iters { + black_box(hypercore.clear(i, 1).await.unwrap()); + } + start.elapsed() } -criterion_group!(benches, create_disk, write_disk, read_disk, clear_disk); +criterion_group!( + benches, + bench_create_disk, + bench_write_disk, + bench_read_disk, + bench_clear_disk +); criterion_main!(benches); diff --git a/benches/memory.rs b/benches/memory.rs index 959b7a76..b439b1e1 100644 --- a/benches/memory.rs +++ b/benches/memory.rs @@ -1,10 +1,23 @@ -use std::time::Instant; +use std::time::{Duration, Instant}; +#[cfg(feature = "async-std")] use criterion::async_executor::AsyncStdExecutor; use criterion::{black_box, criterion_group, criterion_main, Criterion}; use hypercore::{Hypercore, HypercoreBuilder, HypercoreError, Storage}; use random_access_memory::RandomAccessMemory; +fn bench_create_memory(c: &mut Criterion) { + #[cfg(feature = "async-std")] + c.bench_function("create memory", |b| { + b.to_async(AsyncStdExecutor).iter(|| create_hypercore(1024)); + }); + #[cfg(feature = "tokio")] + c.bench_function("create memory", |b| { + let rt = tokio::runtime::Runtime::new().unwrap(); + b.to_async(&rt).iter(|| create_hypercore(1024)); + }); +} + #[cfg(feature = "cache")] async fn create_hypercore( page_size: usize, @@ -32,69 +45,84 @@ async fn create_hypercore( HypercoreBuilder::new(storage).build().await } -fn create_memory(c: &mut Criterion) { - c.bench_function("create_memory", move |b| { - b.to_async(AsyncStdExecutor).iter(|| create_hypercore(1024)); +fn bench_write_memory(c: &mut Criterion) { + #[cfg(feature = "async-std")] + c.bench_function("write memory", |b| { + b.to_async(AsyncStdExecutor).iter_custom(write_memory); + }); + #[cfg(feature = "tokio")] + c.bench_function("write memory", |b| { + let rt = tokio::runtime::Runtime::new().unwrap(); + b.to_async(&rt).iter_custom(write_memory); }); } -fn write_memory(c: &mut Criterion) { - c.bench_function("write_memory", move |b| { - b.to_async(AsyncStdExecutor) - .iter_custom(|iters| async move { - let mut hypercore = create_hypercore(1024).await.unwrap(); - let data = Vec::from("hello"); - let start = Instant::now(); - for _ in 0..iters { - black_box(hypercore.append(&data).await.unwrap()); - } - start.elapsed() - }); +async fn write_memory(iters: u64) -> Duration { + let mut hypercore = create_hypercore(1024).await.unwrap(); + let data = Vec::from("hello"); + let start = Instant::now(); + for _ in 0..iters { + black_box(hypercore.append(&data).await.unwrap()); + } + start.elapsed() +} + +fn bench_read_memory(c: &mut Criterion) { + #[cfg(feature = "async-std")] + c.bench_function("read memory", |b| { + b.to_async(AsyncStdExecutor).iter_custom(read_memory); + }); + #[cfg(feature = "tokio")] + c.bench_function("read memory", |b| { + let rt = tokio::runtime::Runtime::new().unwrap(); + b.to_async(&rt).iter_custom(read_memory); }); } -fn read_memory(c: &mut Criterion) { - c.bench_function("read_memory", move |b| { - b.to_async(AsyncStdExecutor) - .iter_custom(|iters| async move { - let mut hypercore = create_hypercore(1024).await.unwrap(); - let data = Vec::from("hello"); - for _ in 0..iters { - hypercore.append(&data).await.unwrap(); - } - let start = Instant::now(); - for i in 0..iters { - black_box(hypercore.get(i).await.unwrap()); - } - start.elapsed() - }); +async fn read_memory(iters: u64) -> Duration { + let mut hypercore = create_hypercore(1024).await.unwrap(); + let data = Vec::from("hello"); + for _ in 0..iters { + hypercore.append(&data).await.unwrap(); + } + let start = Instant::now(); + for i in 0..iters { + black_box(hypercore.get(i).await.unwrap()); + } + start.elapsed() +} + +fn bench_clear_memory(c: &mut Criterion) { + #[cfg(feature = "async-std")] + c.bench_function("clear memory", |b| { + b.to_async(AsyncStdExecutor).iter_custom(clear_memory); + }); + #[cfg(feature = "tokio")] + c.bench_function("clear memory", |b| { + let rt = tokio::runtime::Runtime::new().unwrap(); + b.to_async(&rt).iter_custom(clear_memory); }); } #[allow(clippy::unit_arg)] -fn clear_memory(c: &mut Criterion) { - c.bench_function("clear_memory", move |b| { - b.to_async(AsyncStdExecutor) - .iter_custom(|iters| async move { - let mut hypercore = create_hypercore(1024).await.unwrap(); - let data = Vec::from("hello"); - for _ in 0..iters { - hypercore.append(&data).await.unwrap(); - } - let start = Instant::now(); - for i in 0..iters { - black_box(hypercore.clear(i, 1).await.unwrap()); - } - start.elapsed() - }); - }); +async fn clear_memory(iters: u64) -> Duration { + let mut hypercore = create_hypercore(1024).await.unwrap(); + let data = Vec::from("hello"); + for _ in 0..iters { + hypercore.append(&data).await.unwrap(); + } + let start = Instant::now(); + for i in 0..iters { + black_box(hypercore.clear(i, 1).await.unwrap()); + } + start.elapsed() } criterion_group!( benches, - create_memory, - write_memory, - read_memory, - clear_memory + bench_create_memory, + bench_write_memory, + bench_read_memory, + bench_clear_memory ); criterion_main!(benches); From 0b26693f5fda3e57257939919798393c6d5cf5b0 Mon Sep 17 00:00:00 2001 From: Timo Tiuraniemi Date: Tue, 3 Oct 2023 16:31:03 +0300 Subject: [PATCH 146/157] Added examples of memory and disk storages with basic functionality --- examples/async.rs | 3 -- examples/disk.rs | 79 ++++++++++++++++++++++++++++++++++++++++++++ examples/iter.rs | 80 --------------------------------------------- examples/main.rs | 4 --- examples/memory.rs | 50 ++++++++++++++++++++++++++++ tests/common/mod.rs | 12 +++---- 6 files changed, 135 insertions(+), 93 deletions(-) delete mode 100644 examples/async.rs create mode 100644 examples/disk.rs delete mode 100644 examples/iter.rs delete mode 100644 examples/main.rs create mode 100644 examples/memory.rs diff --git a/examples/async.rs b/examples/async.rs deleted file mode 100644 index f0610c2c..00000000 --- a/examples/async.rs +++ /dev/null @@ -1,3 +0,0 @@ -fn main() { - unimplemented!() -} diff --git a/examples/disk.rs b/examples/disk.rs new file mode 100644 index 00000000..ecfbb682 --- /dev/null +++ b/examples/disk.rs @@ -0,0 +1,79 @@ +#[cfg(feature = "async-std")] +use async_std::main as async_main; +use hypercore::{HypercoreBuilder, Storage}; +use tempfile::Builder; +#[cfg(feature = "tokio")] +use tokio::main as async_main; + +#[async_main] +async fn main() { + // For the purposes of this example, first create a + // temporary directory to hold hypercore. + let dir = Builder::new() + .prefix("examples_disk") + .tempdir() + .unwrap() + .into_path(); + + // Create a disk storage, overwriting existing values. + let overwrite = true; + let storage = Storage::new_disk(&dir, overwrite) + .await + .expect("Could not create disk storage"); + + // Build a new disk hypercore + let mut hypercore = HypercoreBuilder::new(storage) + .build() + .await + .expect("Could not create disk hypercore"); + + // Append values to the hypercore + hypercore.append(b"Hello, ").await.unwrap(); + hypercore.append(b"from ").await.unwrap(); + + // Close hypercore + drop(hypercore); + + // Open hypercore again from same directory, not + // overwriting. + let overwrite = false; + let storage = Storage::new_disk(&dir, overwrite) + .await + .expect("Could not open existing disk storage"); + let mut hypercore = HypercoreBuilder::new(storage) + .open(true) + .build() + .await + .expect("Could not open disk hypercore"); + + // Append new values to the hypercore + hypercore.append(b"disk hypercore!").await.unwrap(); + + // Add three values and clear the first two + let batch: &[&[u8]] = &[ + b"first value to clear", + b"second value to clear", + b"third value to keep", + ]; + let new_length = hypercore.append_batch(batch).await.unwrap().length; + hypercore + .clear(new_length - 3, new_length - 1) + .await + .unwrap(); + + // The two values return None, but the last one returns correctly + assert!(hypercore.get(3).await.unwrap().is_none()); + assert!(hypercore.get(4).await.unwrap().is_none()); + assert_eq!( + hypercore.get(5).await.unwrap().unwrap(), + b"third value to keep" + ); + + // Print the first three values, converting binary back to string + println!( + "{}{}{}", + String::from_utf8(hypercore.get(0).await.unwrap().unwrap()).unwrap(), + String::from_utf8(hypercore.get(1).await.unwrap().unwrap()).unwrap(), + String::from_utf8(hypercore.get(2).await.unwrap().unwrap()).unwrap() + ); // prints "Hello, from disk hypercore!" +} diff --git a/examples/iter.rs b/examples/iter.rs deleted file mode 100644 index f31e0f6c..00000000 --- a/examples/iter.rs +++ /dev/null @@ -1,80 +0,0 @@ -use std::iter; - -#[derive(Debug)] -struct Book { - pub title: String, -} - -#[derive(Debug)] -struct BookShelf { - pub books: Vec, -} - -#[derive(Debug)] -struct BookShelfIterator<'b> { - /// Keeps track which index we're currently at. - pub cursor: u64, - /// Borrow of the Bookshelf we're going to iterate over. - pub inner: &'b BookShelf, -} - -impl BookShelf { - /// Return an iterator over all values. - pub fn iter(&self) -> BookShelfIterator<'_> { - BookShelfIterator { - inner: self, - cursor: 0, - } - } -} - -impl<'b> iter::Iterator for BookShelfIterator<'b> { - type Item = &'b Book; - - fn next(&mut self) -> Option { - let cursor = self.cursor; - self.cursor += 1; - - if cursor >= self.inner.books.len() as u64 { - None - } else { - Some(&self.inner.books[cursor as usize]) - } - } -} - -impl<'b> iter::IntoIterator for &'b BookShelf { - type Item = &'b Book; - type IntoIter = BookShelfIterator<'b>; - - fn into_iter(self) -> Self::IntoIter { - Self::IntoIter { - cursor: 0, - inner: self, - } - } -} - -fn main() { - let library = BookShelf { - books: vec![ - Book { - title: "Das Kapital I".into(), - }, - Book { - title: "Das Kapital II".into(), - }, - Book { - title: "Das Kapital III".into(), - }, - ], - }; - - for book in library.iter() { - println!("book {}", book.title); - } - - for book in &library { - println!("book {}", book.title); - } -} diff --git a/examples/main.rs b/examples/main.rs deleted file mode 100644 index d7f45deb..00000000 --- a/examples/main.rs +++ /dev/null @@ -1,4 +0,0 @@ -#[async_std::main] -async fn main() { - unimplemented!(); -} diff --git a/examples/memory.rs b/examples/memory.rs new file mode 100644 index 00000000..9cc43970 --- /dev/null +++ b/examples/memory.rs @@ -0,0 +1,50 @@ +#[cfg(feature = "async-std")] +use async_std::main as async_main; +use hypercore::{HypercoreBuilder, Storage}; +#[cfg(feature = "tokio")] +use tokio::main as async_main; + +#[async_main] +async fn main() { + // Create a memory storage + let storage = Storage::new_memory() + .await + .expect("Could not create memory storage"); + + // Build hypercore + let mut hypercore = HypercoreBuilder::new(storage) + .build() + .await + .expect("Could not create memory hypercore"); + + // Append values + hypercore.append(b"Hello, ").await.unwrap(); + hypercore.append(b"from memory hypercore!").await.unwrap(); + + // Add three values and clear the first two + let batch: &[&[u8]] = &[ + b"first value to clear", + b"second value to clear", + b"third value to keep", + ]; + let new_length = hypercore.append_batch(batch).await.unwrap().length; + hypercore + .clear(new_length - 3, new_length - 1) + .await + .unwrap(); + + // The two values return None, but the last one returns correctly + assert!(hypercore.get(2).await.unwrap().is_none()); + assert!(hypercore.get(3).await.unwrap().is_none()); + assert_eq!( + hypercore.get(4).await.unwrap().unwrap(), + b"third value to keep" + ); + + // Print values, converting binary back to string + println!( + "{}{}", + String::from_utf8(hypercore.get(0).await.unwrap().unwrap()).unwrap(), + String::from_utf8(hypercore.get(1).await.unwrap().unwrap()).unwrap() + ); // prints "Hello, from memory hypercore!" +} diff --git a/tests/common/mod.rs b/tests/common/mod.rs index 1814262f..fbe8616c 100644 --- a/tests/common/mod.rs +++ b/tests/common/mod.rs @@ -52,10 +52,10 @@ pub async fn open_hypercore(work_dir: &str) -> Result HypercoreHash { - let bitfield = hash_file(format!("{}/bitfield", dir)); - let data = hash_file(format!("{}/data", dir)); - let oplog = hash_file(format!("{}/oplog", dir)); - let tree = hash_file(format!("{}/tree", dir)); + let bitfield = hash_file(format!("{dir}/bitfield")); + let data = hash_file(format!("{dir}/data")); + let oplog = hash_file(format!("{dir}/oplog")); + let tree = hash_file(format!("{dir}/tree")); HypercoreHash { bitfield, data, @@ -73,13 +73,13 @@ pub fn hash_file(file: String) -> Option { let mut file = std::fs::File::open(path).unwrap(); std::io::copy(&mut file, &mut hasher).unwrap(); let hash_bytes = hasher.finalize(); - let hash = format!("{:X}", hash_bytes); + let hash = format!("{hash_bytes:X}"); // Empty file has this hash, don't make a difference between missing and empty file. Rust // is much easier and performant to write if the empty file is created. if hash == *"E3B0C44298FC1C149AFBF4C8996FB92427AE41E4649B934CA495991B7852B855" { None } else { - Some(format!("{:X}", hash_bytes)) + Some(format!("{hash_bytes:X}")) } } } From 66633bf78581b0464ea4ed75a317e43f7ff54754 Mon Sep 17 00:00:00 2001 From: Timo Tiuraniemi Date: Wed, 4 Oct 2023 11:18:51 +0300 Subject: [PATCH 147/157] Change missing_nodes to accept hypercore index Add missing_nodes_from_merkle_tree_index for a possibility to evaluate also odd merkle tree indices. Also add replication example. --- Cargo.toml | 7 ++- examples/disk.rs | 17 ++++-- examples/memory.rs | 15 ++++-- examples/replication.rs | 116 ++++++++++++++++++++++++++++++++++++++++ src/core.rs | 12 ++++- src/tree/merkle_tree.rs | 2 +- 6 files changed, 158 insertions(+), 11 deletions(-) create mode 100644 examples/replication.rs diff --git a/Cargo.toml b/Cargo.toml index 3e843649..a7504883 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,12 +1,15 @@ [package] name = "hypercore" -version = "0.12.0-alpha.7" +version = "0.12.0-alpha.8" license = "MIT OR Apache-2.0" description = "Secure, distributed, append-only log" documentation = "https://docs.rs/hypercore" repository = "https://github.com/datrs/hypercore" readme = "README.md" -authors = ["Yoshua Wuyts "] +authors = [ + "Yoshua Wuyts ", + "Timo Tiuraniemi " +] keywords = ["dat", "p2p", "stream", "feed", "merkle"] categories = [ "asynchronous", diff --git a/examples/disk.rs b/examples/disk.rs index ecfbb682..99990897 100644 --- a/examples/disk.rs +++ b/examples/disk.rs @@ -1,10 +1,11 @@ #[cfg(feature = "async-std")] use async_std::main as async_main; -use hypercore::{HypercoreBuilder, Storage}; +use hypercore::{HypercoreBuilder, HypercoreError, Storage}; use tempfile::Builder; #[cfg(feature = "tokio")] use tokio::main as async_main; +/// Example about using an in-memory hypercore. #[async_main] async fn main() { // For the purposes of this example, first create a @@ -72,8 +73,16 @@ async fn main() { // Print the first three values, converting binary back to string println!( "{}{}{}", - String::from_utf8(hypercore.get(0).await.unwrap().unwrap()).unwrap(), - String::from_utf8(hypercore.get(1).await.unwrap().unwrap()).unwrap(), - String::from_utf8(hypercore.get(2).await.unwrap().unwrap()).unwrap() + format_res(hypercore.get(0).await), + format_res(hypercore.get(1).await), + format_res(hypercore.get(2).await) ); // prints "Hello, from disk hypercore!" } + +fn format_res(res: Result>, HypercoreError>) -> String { + match res { + Ok(Some(bytes)) => String::from_utf8(bytes).expect("Shouldn't fail in example"), + Ok(None) => "Got None in feed".to_string(), + Err(e) => format!("Error getting value from feed, reason = {e:?}"), + } +} diff --git a/examples/memory.rs b/examples/memory.rs index 9cc43970..a510ed6d 100644 --- a/examples/memory.rs +++ b/examples/memory.rs @@ -1,9 +1,10 @@ #[cfg(feature = "async-std")] use async_std::main as async_main; -use hypercore::{HypercoreBuilder, Storage}; +use hypercore::{HypercoreBuilder, HypercoreError, Storage}; #[cfg(feature = "tokio")] use tokio::main as async_main; +/// Example about using an in-memory hypercore. #[async_main] async fn main() { // Create a memory storage @@ -44,7 +45,15 @@ async fn main() { // Print values, converting binary back to string println!( "{}{}", - String::from_utf8(hypercore.get(0).await.unwrap().unwrap()).unwrap(), - String::from_utf8(hypercore.get(1).await.unwrap().unwrap()).unwrap() + format_res(hypercore.get(0).await), + format_res(hypercore.get(1).await) ); // prints "Hello, from memory hypercore!" } + +fn format_res(res: Result>, HypercoreError>) -> String { + match res { + Ok(Some(bytes)) => String::from_utf8(bytes).expect("Shouldn't fail in example"), + Ok(None) => "Got None in feed".to_string(), + Err(e) => format!("Error getting value from feed, reason = {e:?}"), + } +} diff --git a/examples/replication.rs b/examples/replication.rs new file mode 100644 index 00000000..52c205ac --- /dev/null +++ b/examples/replication.rs @@ -0,0 +1,116 @@ +#[cfg(feature = "async-std")] +use async_std::main as async_main; +use hypercore::{ + Hypercore, HypercoreBuilder, HypercoreError, PartialKeypair, RequestBlock, RequestUpgrade, + Storage, +}; +use random_access_disk::RandomAccessDisk; +use random_access_memory::RandomAccessMemory; +use tempfile::Builder; +#[cfg(feature = "tokio")] +use tokio::main as async_main; + +/// Example on how to replicate a (disk) hypercore to another (memory) hypercore. +/// NB: The replication functions used here are low-level, built for use in the wire +/// protocol. +#[async_main] +async fn main() { + // For the purposes of this example, first create a + // temporary directory to hold hypercore. + let dir = Builder::new() + .prefix("examples_replication") + .tempdir() + .unwrap() + .into_path(); + + // Create a disk storage, overwriting existing values. + let overwrite = true; + let storage = Storage::new_disk(&dir, overwrite) + .await + .expect("Could not create disk storage"); + + // Build a new disk hypercore + let mut origin_hypercore = HypercoreBuilder::new(storage) + .build() + .await + .expect("Could not create disk hypercore"); + + // Append values to the hypercore + let batch: &[&[u8]] = &[b"Hello, ", b"from ", b"replicated ", b"hypercore!"]; + origin_hypercore.append_batch(batch).await.unwrap(); + + // Store the public key + let origin_public_key = origin_hypercore.key_pair().public; + + // Create a peer of the origin hypercore using the public key + let mut replicated_hypercore = HypercoreBuilder::new( + Storage::new_memory() + .await + .expect("Could not create memory storage"), + ) + .key_pair(PartialKeypair { + public: origin_public_key, + secret: None, + }) + .build() + .await + .expect("Could not create memory hypercore"); + + // Replicate the four values in random order + replicate_index(&mut origin_hypercore, &mut replicated_hypercore, 3).await; + replicate_index(&mut origin_hypercore, &mut replicated_hypercore, 0).await; + replicate_index(&mut origin_hypercore, &mut replicated_hypercore, 2).await; + replicate_index(&mut origin_hypercore, &mut replicated_hypercore, 1).await; + + // Print values from replicated hypercore, converting binary back to string + println!( + "{}{}{}{}", + format_res(replicated_hypercore.get(0).await), + format_res(replicated_hypercore.get(1).await), + format_res(replicated_hypercore.get(2).await), + format_res(replicated_hypercore.get(3).await) + ); // prints "Hello, from replicated hypercore!" +} + +async fn replicate_index( + origin_hypercore: &mut Hypercore, + replicated_hypercore: &mut Hypercore, + request_index: u64, +) { + let missing_nodes = origin_hypercore + .missing_nodes(request_index) + .await + .expect("Could not get missing nodes"); + let upgrade_start = replicated_hypercore.info().contiguous_length; + let upgrade_length = origin_hypercore.info().contiguous_length - upgrade_start; + + let proof = origin_hypercore + .create_proof( + Some(RequestBlock { + index: request_index, + nodes: missing_nodes, + }), + None, + None, + Some(RequestUpgrade { + start: upgrade_start, + length: upgrade_length, + }), + ) + .await + .expect("Creating proof error") + .expect("Could not get proof"); + // Then the proof is verified and applied to the replicated party. + assert!(replicated_hypercore + .verify_and_apply_proof(&proof) + .await + .expect("Verifying and applying proof failed")); +} + +fn format_res(res: Result>, HypercoreError>) -> String { + match res { + Ok(Some(bytes)) => String::from_utf8(bytes).expect("Shouldn't fail in example"), + Ok(None) => "Got None in feed".to_string(), + Err(e) => format!("Error getting value from feed, reason = {e:?}"), + } +} diff --git a/src/core.rs b/src/core.rs index 427b253a..9885fe3b 100644 --- a/src/core.rs +++ b/src/core.rs @@ -550,7 +550,17 @@ where /// Used to fill the nodes field of a `RequestBlock` during /// synchronization. #[instrument(err, skip(self))] - pub async fn missing_nodes(&mut self, merkle_tree_index: u64) -> Result { + pub async fn missing_nodes(&mut self, index: u64) -> Result { + self.missing_nodes_from_merkle_tree_index(index * 2).await + } + + /// Get missing nodes using a merkle tree index. Advanced variant of missing_nodex + /// that allow for special cases of searching directly from the merkle tree. + #[instrument(err, skip(self))] + pub async fn missing_nodes_from_merkle_tree_index( + &mut self, + merkle_tree_index: u64, + ) -> Result { match self.tree.missing_nodes(merkle_tree_index, None)? { Either::Right(value) => Ok(value), Either::Left(instructions) => { diff --git a/src/tree/merkle_tree.rs b/src/tree/merkle_tree.rs index 684eed61..c9579199 100644 --- a/src/tree/merkle_tree.rs +++ b/src/tree/merkle_tree.rs @@ -328,7 +328,7 @@ impl MerkleTree { /// Creates valueless proof from requests. /// TODO: This is now just a clone of javascript's - /// https://github.com/hypercore-protocol/hypercore/blob/7e30a0fe353c70ada105840ec1ead6627ff521e7/lib/merkle-tree.js#L604 + /// https://github.com/holepunchto/hypercore/blob/9ce03363cb8938dbab53baba7d7cc9dde0508a7e/lib/merkle-tree.js#L1181 /// The implementation should be rewritten to make it clearer. pub(crate) fn create_valueless_proof( &mut self, From 0041eeb534feb97e0a3f775463e99e252e199c11 Mon Sep 17 00:00:00 2001 From: Timo Tiuraniemi Date: Wed, 4 Oct 2023 12:28:48 +0300 Subject: [PATCH 148/157] Fix WASM build --- Cargo.toml | 1 + src/storage/mod.rs | 1 + 2 files changed, 2 insertions(+) diff --git a/Cargo.toml b/Cargo.toml index a7504883..88746938 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -24,6 +24,7 @@ edition = "2018" blake2 = "0.10.6" byteorder = "1.3.4" ed25519-dalek = { version = "2.0.0", features = ["rand_core"] } +getrandom = { version = "0.2", features = ["js"] } thiserror = "1" tracing = "0.1" compact-encoding = "1" diff --git a/src/storage/mod.rs b/src/storage/mod.rs index c989afe0..7eb3776d 100644 --- a/src/storage/mod.rs +++ b/src/storage/mod.rs @@ -6,6 +6,7 @@ use random_access_disk::RandomAccessDisk; use random_access_memory::RandomAccessMemory; use random_access_storage::{RandomAccess, RandomAccessError}; use std::fmt::Debug; +#[cfg(not(target_arch = "wasm32"))] use std::path::PathBuf; use tracing::instrument; From 4ca4037661ebb85ccbacccd04adf82d86e4ff573 Mon Sep 17 00:00:00 2001 From: Timo Tiuraniemi Date: Wed, 4 Oct 2023 12:42:35 +0300 Subject: [PATCH 149/157] Add basic library level documentation --- Cargo.toml | 1 + src/lib.rs | 62 +++++++++++++++++++++++++++++++++++++++++++++++------- 2 files changed, 55 insertions(+), 8 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 88746938..a55905b9 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -58,6 +58,7 @@ remove_dir_all = "0.7.0" tempfile = "3.1.0" async-std = { version = "1.12.0", features = ["attributes"] } tokio = { version = "1.27.0", default-features = false, features = ["macros", "rt", "rt-multi-thread"] } +tokio-test = "0.4" sha2 = "0.10.2" criterion = { version = "0.4", features = ["async_std", "async_tokio"] } test-log = { version = "0.2.11", default-features = false, features = ["trace"] } diff --git a/src/lib.rs b/src/lib.rs index 16284dee..a403e381 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -7,24 +7,70 @@ #![doc(test(attr(deny(warnings))))] //! ## Introduction +//! //! Hypercore is a secure, distributed append-only log. Built for sharing //! large datasets and streams of real time data as part of the [Dat] project. -//! This is a rust port of [the original node version][dat-node] -//! aiming for interoperability. The primary way to use this crate is through the [Feed] struct. +//! This is a rust port of [the original Javascript version][holepunch-hypercore] +//! aiming for interoperability with LTS version. The primary way to use this +//! crate is through the [Hypercore] struct, which can be created using the +//! [HypercoreBuilder]. +//! +//! This crate supports WASM with `cargo build --target=wasm32-unknown-unknown`. +//! +//! ## Features +//! +//! ### `sparse` (default) +//! +//! When using disk storage, clearing values may create sparse files. On by default. +//! +//! ### `async-std` (default) +//! +//! Use the async-std runtime, on by default. Either this or `tokio` is mandatory. +//! +//! ### `tokio` +//! +//! Use the tokio runtime. Either this or `async_std` is mandatory. +//! +//! ### `cache` +//! +//! Use a moka cache for merkle tree nodes to speed-up reading. //! //! ## Example //! ```rust -//! # fn main() -> Result<(), Box> { +//! # #[cfg(feature = "tokio")] +//! # tokio_test::block_on(async { +//! # example().await; +//! # }); +//! # #[cfg(feature = "async-std")] //! # async_std::task::block_on(async { -//! // unimplemented -//! Ok(()) -//! # }) +//! # example().await; +//! # }); +//! # async fn example() { +//! use hypercore::{HypercoreBuilder, Storage}; +//! +//! // Create an in-memory hypercore using a builder +//! let mut hypercore = HypercoreBuilder::new(Storage::new_memory().await.unwrap()) +//! .build() +//! .await +//! .unwrap(); +//! +//! // Append entries to the log +//! hypercore.append(b"Hello, ").await.unwrap(); +//! hypercore.append(b"world!").await.unwrap(); +//! +//! // Read entries from the log +//! assert_eq!(hypercore.get(0).await.unwrap().unwrap(), b"Hello, "); +//! assert_eq!(hypercore.get(1).await.unwrap().unwrap(), b"world!"); //! # } //! ``` //! -//! [dat-node]: https://github.com/mafintosh/hypercore +//! Find more examples in the [examples] folder. +//! //! [Dat]: https://github.com/datrs -//! [Feed]: crate::feed::Feed +//! [holepunch-hypercore]: https://github.com/holepunchto/hypercore +//! [Hypercore]: crate::core::Hypercore +//! [HypercoreBuilder]: crate::builder::HypercoreBuilder +//! [examples]: https://github.com/datrs/hypercore/tree/master/examples pub mod encoding; pub mod prelude; From bdc843a465c120c94f6ae69a318d1419fa9a3283 Mon Sep 17 00:00:00 2001 From: Timo Tiuraniemi Date: Wed, 4 Oct 2023 13:38:52 +0300 Subject: [PATCH 150/157] Fix broken tests from missing_nodes change --- src/core.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/core.rs b/src/core.rs index 9885fe3b..fe49e9a2 100644 --- a/src/core.rs +++ b/src/core.rs @@ -1067,7 +1067,7 @@ mod tests { ) .await?; let index = 6; - let nodes = clone.missing_nodes(index * 2).await?; + let nodes = clone.missing_nodes(index).await?; let proof = main .create_proof( None, @@ -1090,7 +1090,7 @@ mod tests { // Fetch data for index 6 and verify it is found let index = 6; - let nodes = clone.missing_nodes(index * 2).await?; + let nodes = clone.missing_nodes(index).await?; let proof = main .create_proof(Some(RequestBlock { index, nodes }), None, None, None) .await? From e7aabf9a8f79b18889e924726ff2c132f6d2b90d Mon Sep 17 00:00:00 2001 From: Timo Tiuraniemi Date: Wed, 4 Oct 2023 13:39:18 +0300 Subject: [PATCH 151/157] Fix README and introduce GHA --- .github/workflows/ci.yml | 141 +++++++++++++++++++++++++++++++++++++++ README.md | 41 +++++++----- 2 files changed, 165 insertions(+), 17 deletions(-) create mode 100644 .github/workflows/ci.yml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 00000000..696188d2 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,141 @@ +name: 'CI' +on: + pull_request: + push: + branches: + # FIXME: Temporarily test on v10 branch + - v10 + +env: + RUST_BACKTRACE: 1 + CARGO_TERM_COLOR: always + +jobs: + ci-pass: + name: CI is green + runs-on: ubuntu-latest + needs: + - test-linux + - test-windows + - test-macos + - build-extra + - lint + steps: + - run: exit 0 + + test-linux: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v3 + - uses: dtolnay/rust-toolchain@stable + with: + components: rustfmt + - name: Run tests + run: | + cargo check --no-default-features --features tokio + cargo check --no-default-features --features tokio,sparse + cargo check --no-default-features --features tokio,sparse,cache + cargo check --no-default-features --features async-std + cargo check --no-default-features --features async-std,sparse + cargo check --no-default-features --features async-std,sparse,cache + cargo test --no-default-features --features js_interop_tests,tokio + cargo test --no-default-features --features js_interop_tests,tokio,sparse + cargo test --no-default-features --features js_interop_tests,tokio,sparse,cache + cargo test --no-default-features --features js_interop_tests,async-std + cargo test --no-default-features --features js_interop_tests,async-std,sparse + cargo test --no-default-features --features js_interop_tests,async-std,sparse,cache + + test-windows: + runs-on: windows-latest + + steps: + - uses: actions/checkout@v3 + - uses: dtolnay/rust-toolchain@stable + with: + components: rustfmt + - name: Run tests + run: | + cargo check --no-default-features --features tokio + cargo check --no-default-features --features tokio,sparse + cargo check --no-default-features --features tokio,sparse,cache + cargo check --no-default-features --features async-std + cargo check --no-default-features --features async-std,sparse + cargo check --no-default-features --features async-std,sparse,cache + cargo test --no-default-features --features js_interop_tests,tokio + cargo test --no-default-features --features js_interop_tests,tokio,sparse + cargo test --no-default-features --features js_interop_tests,tokio,sparse,cache + cargo test --no-default-features --features js_interop_tests,async-std + cargo test --no-default-features --features js_interop_tests,async-std,sparse + cargo test --no-default-features --features js_interop_tests,async-std,sparse,cache + + test-macos: + runs-on: macos-latest + + steps: + - uses: actions/checkout@v3 + - uses: dtolnay/rust-toolchain@stable + with: + components: rustfmt + - name: Run tests + run: | + cargo check --no-default-features --features tokio + cargo check --no-default-features --features tokio,sparse + cargo check --no-default-features --features tokio,sparse,cache + cargo check --no-default-features --features async-std + cargo check --no-default-features --features async-std,sparse + cargo check --no-default-features --features async-std,sparse,cache + cargo test --no-default-features --features js_interop_tests,tokio + cargo test --no-default-features --features js_interop_tests,tokio,sparse + cargo test --no-default-features --features js_interop_tests,tokio,sparse,cache + cargo test --no-default-features --features js_interop_tests,async-std + cargo test --no-default-features --features js_interop_tests,async-std,sparse + cargo test --no-default-features --features js_interop_tests,async-std,sparse,cache + + build-extra: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v3 + - uses: dtolnay/rust-toolchain@stable + with: + targets: wasm32-unknown-unknown + - name: Build WASM + run: | + cargo build --target=wasm32-unknown-unknown --no-default-features --features tokio + cargo build --target=wasm32-unknown-unknown --no-default-features --features async-std + - name: Build benches + run: | + cargo build --benches --no-default-features --features tokio + cargo build --benches --no-default-features --features async-std + - name: Build release + run: | + cargo build --release --no-default-features --features tokio + cargo build --release --no-default-features --features tokio,sparse + cargo build --release --no-default-features --features tokio,sparse,cache + cargo build --release --no-default-features --features async-std + cargo build --release --no-default-features --features async-std,sparse + cargo build --release --no-default-features --features async-std,sparse,cache + - name: Run examples + run: | + cargo run --no-default-features --features tokio --example disk + cargo run --no-default-features --features async-std --example disk + cargo run --no-default-features --features tokio --example memory + cargo run --no-default-features --features async-std --example memory + cargo run --no-default-features --features tokio --example replication + cargo run --no-default-features --features async-std --example replication + + lint: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v3 + - uses: dtolnay/rust-toolchain@stable + with: + components: rustfmt, clippy + - uses: actions-rs/clippy-check@v1 + with: + token: ${{ secrets.GITHUB_TOKEN }} + - name: Format check + run: | + cargo fmt -- --check diff --git a/README.md b/README.md index fc0de20d..8a712032 100644 --- a/README.md +++ b/README.md @@ -1,25 +1,33 @@ -# hypercore +# Hypercore [![crates.io version][1]][2] [![build status][3]][4] [![downloads][5]][6] [![docs.rs docs][7]][8] -WIP. Secure, distributed, append-only log structure. Adapted from -[mafintosh/hypercore](https://github.com/mafintosh/hypercore). +Hypercore is a secure, distributed append-only log. This crate is a limited Rust +port of the original Javascript +[holepunchto/hypercore](https://github.com/holepunchto/hypercore). The goal is to +maintain binary compatibility with the LTS version with regards to disk storage. + +See [hypercore-protocol-rs](https://github.com/datrs/hypercore-protocol-rs) for the +corresponding wire protocol implementation. - [Documentation][8] - [Crates.io][2] -**NOTE**: The master branch currently only works with the old hypercore version 9. -For ongoing work to support the latest version 10 of hypercore [see the v10 branch](https://github.com/datrs/hypercore/tree/v10). - ## Usage ```rust -let mut feed = hypercore::open("./feed.db").await?; +// Create an in-memory hypercore using a builder +let mut hypercore = HypercoreBuilder::new(Storage::new_memory().await.unwrap()) + .build() + .await + .unwrap(); -feed.append(b"hello").await?; -feed.append(b"world").await?; +// Append entries to the log +hypercore.append(b"Hello, ").await.unwrap(); +hypercore.append(b"world!").await.unwrap(); -assert_eq!(feed.get(0).await?, Some(b"hello".to_vec())); -assert_eq!(feed.get(1).await?, Some(b"world".to_vec())); +// Read entries from the log +assert_eq!(hypercore.get(0).await.unwrap().unwrap(), b"Hello, "); +assert_eq!(hypercore.get(1).await.unwrap().unwrap(), b"world!"); ``` ## Installation @@ -27,8 +35,10 @@ assert_eq!(feed.get(1).await?, Some(b"world".to_vec())); $ cargo add hypercore ``` +Find more examples in the [examples](./examples) folder. + ## Safety -This crate uses ``#![deny(unsafe_code)]`` to ensure everything is implemented in +This crate uses ``#![forbid(unsafe_code)]`` to ensure everythong is implemented in 100% Safe Rust. ## Contributing @@ -38,16 +48,13 @@ look at some of these issues: - [Issues labeled "good first issue"][good-first-issue] - [Issues labeled "help wanted"][help-wanted] -## References -- [github.com/mafintosh/hypercore](https://github.com/mafintosh/hypercore) - ## License [MIT](./LICENSE-MIT) OR [Apache-2.0](./LICENSE-APACHE) [1]: https://img.shields.io/crates/v/hypercore.svg?style=flat-square [2]: https://crates.io/crates/hypercore -[3]: https://img.shields.io/travis/datrs/hypercore/master.svg?style=flat-square -[4]: https://travis-ci.org/datrs/hypercore +[3]: https://github.com/datrs/hypercore/actions/workflows/ci.yml/badge.svg +[4]: https://github.com/datrs/hypercore/actions [5]: https://img.shields.io/crates/d/hypercore.svg?style=flat-square [6]: https://crates.io/crates/hypercore [7]: https://img.shields.io/badge/docs-latest-blue.svg?style=flat-square From 111c9a87ccab4e969d0a9c90fcc4c8f458f2f067 Mon Sep 17 00:00:00 2001 From: Timo Tiuraniemi Date: Wed, 4 Oct 2023 13:41:22 +0300 Subject: [PATCH 152/157] Use 2021 Rust edition --- Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index a55905b9..f9e4ef0e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -18,7 +18,7 @@ categories = [ "data-structures", "encoding", ] -edition = "2018" +edition = "2021" [dependencies] blake2 = "0.10.6" From d7c2fa8d4cd4503f81f45f68187516af49602cf2 Mon Sep 17 00:00:00 2001 From: Timo Tiuraniemi Date: Wed, 4 Oct 2023 14:00:26 +0300 Subject: [PATCH 153/157] Don't run JS interoperability tests on Windows which doesn't have node installed --- .github/workflows/ci.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 696188d2..d36fdc09 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -62,12 +62,12 @@ jobs: cargo check --no-default-features --features async-std cargo check --no-default-features --features async-std,sparse cargo check --no-default-features --features async-std,sparse,cache - cargo test --no-default-features --features js_interop_tests,tokio - cargo test --no-default-features --features js_interop_tests,tokio,sparse - cargo test --no-default-features --features js_interop_tests,tokio,sparse,cache - cargo test --no-default-features --features js_interop_tests,async-std - cargo test --no-default-features --features js_interop_tests,async-std,sparse - cargo test --no-default-features --features js_interop_tests,async-std,sparse,cache + cargo test --no-default-features --features tokio + cargo test --no-default-features --features tokio,sparse + cargo test --no-default-features --features tokio,sparse,cache + cargo test --no-default-features --features async-std + cargo test --no-default-features --features async-std,sparse + cargo test --no-default-features --features async-std,sparse,cache test-macos: runs-on: macos-latest From f6fa0f5da2e2f06cb431e4714e24ba0dc12aba01 Mon Sep 17 00:00:00 2001 From: Timo Tiuraniemi Date: Wed, 4 Oct 2023 14:22:12 +0300 Subject: [PATCH 154/157] Fix CI to run on master and add features to README --- .github/workflows/ci.yml | 3 +-- README.md | 16 +++++++++++++++- 2 files changed, 16 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d36fdc09..c557dc4e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -3,8 +3,7 @@ on: pull_request: push: branches: - # FIXME: Temporarily test on v10 branch - - v10 + - master env: RUST_BACKTRACE: 1 diff --git a/README.md b/README.md index 8a712032..9c70fa03 100644 --- a/README.md +++ b/README.md @@ -13,6 +13,20 @@ corresponding wire protocol implementation. - [Documentation][8] - [Crates.io][2] +## Features + +- [x] Create [in-memory](https://github.com/datrs/random-access-memory) and [disk](https://github.com/datrs/random-access-disk) hypercores +- [x] Append to hypercore either a single entry or a batch of entries +- [x] Get entries from hypercore +- [x] Clear range from hypercore, with optional support for sparse files +- [x] Support basic replication by creating proofs in a source hypercore and verifying and applying them to a destination hypercore +- [x] Support `tokio` or `async-std` runtimes +- [x] Support WASM for in-memory storage +- [x] Test Javascript interoperability for supported features +- [x] Add optional read cache +- [ ] Support the new [manifest](https://github.com/holepunchto/hypercore/blob/main/lib/manifest.js) in the wire protocol to remain compatible with upcoming v11 +- [ ] Finalize documentation and release v1.0.0 + ## Usage ```rust // Create an in-memory hypercore using a builder @@ -32,7 +46,7 @@ assert_eq!(hypercore.get(1).await.unwrap().unwrap(), b"world!"); ## Installation ```sh -$ cargo add hypercore +cargo add hypercore ``` Find more examples in the [examples](./examples) folder. From ec61046855e25a1790542aaef752ade1f8c8c718 Mon Sep 17 00:00:00 2001 From: Timo Tiuraniemi Date: Thu, 5 Oct 2023 08:52:49 +0300 Subject: [PATCH 155/157] Fix deps: bump valid and remove unneeded --- Cargo.toml | 38 ++++++++++++++++---------------------- 1 file changed, 16 insertions(+), 22 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index f9e4ef0e..f9e09fd2 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "hypercore" -version = "0.12.0-alpha.8" +version = "0.12.0-alpha.9" license = "MIT OR Apache-2.0" description = "Secure, distributed, append-only log" documentation = "https://docs.rs/hypercore" @@ -21,33 +21,27 @@ categories = [ edition = "2021" [dependencies] -blake2 = "0.10.6" -byteorder = "1.3.4" -ed25519-dalek = { version = "2.0.0", features = ["rand_core"] } +blake2 = "0.10" +byteorder = "1" +ed25519-dalek = { version = "2", features = ["rand_core"] } getrandom = { version = "0.2", features = ["js"] } thiserror = "1" tracing = "0.1" compact-encoding = "1" -flat-tree = "6.0.0" -lazy_static = "1.4.0" -memory-pager = "0.9.0" -merkle-tree-stream = "0.12.0" -pretty-hash = "0.4.1" +flat-tree = "6" +merkle-tree-stream = "0.12" +pretty-hash = "0.4" rand = "0.8" -random-access-memory = "3.0.0" -random-access-storage = "5.0.0" -sha2 = "0.9.2" -sleep-parser = "0.8.0" -sparse-bitfield = "0.11.0" -tree-index = "0.6.0" -bitfield-rle = "0.2.0" -futures = "0.3.4" -crc32fast = "1.3.2" -intmap = "2.0.0" -moka = { version = "0.10.0", optional = true } +random-access-memory = "3" +random-access-storage = "5" +sha2 = "0.10" +futures = "0.3" +crc32fast = "1" +intmap = "2" +moka = { version = "0.12", optional = true, features = ["sync"] } [target.'cfg(not(target_arch = "wasm32"))'.dependencies] -random-access-disk = { version = "3.0.0", default-features = false } +random-access-disk = { version = "3", default-features = false } [dev-dependencies] anyhow = "1.0.70" @@ -59,7 +53,7 @@ tempfile = "3.1.0" async-std = { version = "1.12.0", features = ["attributes"] } tokio = { version = "1.27.0", default-features = false, features = ["macros", "rt", "rt-multi-thread"] } tokio-test = "0.4" -sha2 = "0.10.2" +sha2 = "0.10" criterion = { version = "0.4", features = ["async_std", "async_tokio"] } test-log = { version = "0.2.11", default-features = false, features = ["trace"] } tracing-subscriber = { version = "0.3.16", features = ["env-filter", "fmt"] } From e8024d1470b99f36fdfa878475fde0602d852021 Mon Sep 17 00:00:00 2001 From: Timo Tiuraniemi Date: Thu, 5 Oct 2023 12:34:31 +0300 Subject: [PATCH 156/157] docs: provide commands to run the examples --- README.md | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 9c70fa03..b0e33e0e 100644 --- a/README.md +++ b/README.md @@ -28,6 +28,7 @@ corresponding wire protocol implementation. - [ ] Finalize documentation and release v1.0.0 ## Usage + ```rust // Create an in-memory hypercore using a builder let mut hypercore = HypercoreBuilder::new(Storage::new_memory().await.unwrap()) @@ -44,18 +45,27 @@ assert_eq!(hypercore.get(0).await.unwrap().unwrap(), b"Hello, "); assert_eq!(hypercore.get(1).await.unwrap().unwrap(), b"world!"); ``` +Find more examples in the [examples](./examples) folder, and/or run: + +```bash +cargo run --example memory +cargo run --example disk +cargo run --example replication +``` + ## Installation -```sh + +```bash cargo add hypercore ``` -Find more examples in the [examples](./examples) folder. - ## Safety + This crate uses ``#![forbid(unsafe_code)]`` to ensure everythong is implemented in 100% Safe Rust. ## Contributing + Want to join us? Check out our ["Contributing" guide][contributing] and take a look at some of these issues: @@ -63,6 +73,7 @@ look at some of these issues: - [Issues labeled "help wanted"][help-wanted] ## License + [MIT](./LICENSE-MIT) OR [Apache-2.0](./LICENSE-APACHE) [1]: https://img.shields.io/crates/v/hypercore.svg?style=flat-square From 68ef0e093057057709a8d359ffde18bb943b398d Mon Sep 17 00:00:00 2001 From: Timo Tiuraniemi Date: Thu, 5 Oct 2023 14:29:55 +0300 Subject: [PATCH 157/157] Run benches once during tests, don't just build them --- .github/workflows/ci.yml | 10 ++++++---- README.md | 14 ++++++++++++++ 2 files changed, 20 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c557dc4e..6976e989 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -44,6 +44,8 @@ jobs: cargo test --no-default-features --features js_interop_tests,async-std cargo test --no-default-features --features js_interop_tests,async-std,sparse cargo test --no-default-features --features js_interop_tests,async-std,sparse,cache + cargo test --benches --no-default-features --features tokio + cargo test --benches --no-default-features --features async-std test-windows: runs-on: windows-latest @@ -67,6 +69,8 @@ jobs: cargo test --no-default-features --features async-std cargo test --no-default-features --features async-std,sparse cargo test --no-default-features --features async-std,sparse,cache + cargo test --benches --no-default-features --features tokio + cargo test --benches --no-default-features --features async-std test-macos: runs-on: macos-latest @@ -90,6 +94,8 @@ jobs: cargo test --no-default-features --features js_interop_tests,async-std cargo test --no-default-features --features js_interop_tests,async-std,sparse cargo test --no-default-features --features js_interop_tests,async-std,sparse,cache + cargo test --benches --no-default-features --features tokio + cargo test --benches --no-default-features --features async-std build-extra: runs-on: ubuntu-latest @@ -103,10 +109,6 @@ jobs: run: | cargo build --target=wasm32-unknown-unknown --no-default-features --features tokio cargo build --target=wasm32-unknown-unknown --no-default-features --features async-std - - name: Build benches - run: | - cargo build --benches --no-default-features --features tokio - cargo build --benches --no-default-features --features async-std - name: Build release run: | cargo build --release --no-default-features --features tokio diff --git a/README.md b/README.md index b0e33e0e..a95afaeb 100644 --- a/README.md +++ b/README.md @@ -64,6 +64,20 @@ cargo add hypercore This crate uses ``#![forbid(unsafe_code)]`` to ensure everythong is implemented in 100% Safe Rust. +## Development + +To test interoperability with Javascript, enable the `js_interop_tests` feature: + +```bash +cargo test --features js_interop_tests +``` + +Run benches with: + +```bash +cargo bench +``` + ## Contributing Want to join us? Check out our ["Contributing" guide][contributing] and take a