diff --git a/Cargo.lock b/Cargo.lock index c942516948d9..fe1fafd23ae6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -7574,6 +7574,7 @@ dependencies = [ "reth-primitives", "reth-primitives-traits", "reth-trie", + "reth-trie-common", "revm", "serde", "serde_with", @@ -9350,7 +9351,6 @@ dependencies = [ "alloy-rlp", "alloy-trie", "auto_impl", - "bincode", "criterion", "itertools 0.13.0", "metrics", @@ -9364,9 +9364,7 @@ dependencies = [ "reth-storage-errors", "reth-trie-common", "revm", - "serde", "serde_json", - "serde_with", "tracing", "triehash", ] @@ -9381,6 +9379,7 @@ dependencies = [ "alloy-rlp", "alloy-trie", "arbitrary", + "bincode", "bytes", "criterion", "derive_more 1.0.0", @@ -9395,6 +9394,7 @@ dependencies = [ "revm-primitives", "serde", "serde_json", + "serde_with", ] [[package]] diff --git a/crates/engine/invalid-block-hooks/Cargo.toml b/crates/engine/invalid-block-hooks/Cargo.toml index 462f0762a9e4..e5eb998dc1ff 100644 --- a/crates/engine/invalid-block-hooks/Cargo.toml +++ b/crates/engine/invalid-block-hooks/Cargo.toml @@ -20,7 +20,7 @@ reth-provider.workspace = true reth-revm = { workspace = true, features = ["serde"] } reth-rpc-api = { workspace = true, features = ["client"] } reth-tracing.workspace = true -reth-trie = { workspace = true, features = ["serde"] } +reth-trie.workspace = true # alloy alloy-primitives.workspace = true diff --git a/crates/evm/execution-types/Cargo.toml b/crates/evm/execution-types/Cargo.toml index 4d2d8214ff98..5eb3cc38437f 100644 --- a/crates/evm/execution-types/Cargo.toml +++ b/crates/evm/execution-types/Cargo.toml @@ -12,9 +12,10 @@ workspace = true [dependencies] reth-primitives.workspace = true +reth-primitives-traits.workspace = true reth-execution-errors.workspace = true +reth-trie-common.workspace = true reth-trie.workspace = true -reth-primitives-traits.workspace = true revm.workspace = true @@ -36,17 +37,18 @@ default = ["std"] optimism = ["reth-primitives/optimism", "revm/optimism"] serde = [ "dep:serde", - "reth-trie/serde", + "rand/serde", "revm/serde", "alloy-eips/serde", "alloy-primitives/serde", - "rand/serde", "reth-primitives-traits/serde", + "reth-trie-common/serde", + "reth-trie/serde", ] serde-bincode-compat = [ "reth-primitives/serde-bincode-compat", "reth-primitives-traits/serde-bincode-compat", - "reth-trie/serde-bincode-compat", + "reth-trie-common/serde-bincode-compat", "serde_with", "alloy-eips/serde-bincode-compat", ] diff --git a/crates/evm/execution-types/src/chain.rs b/crates/evm/execution-types/src/chain.rs index 6aed8422bc33..e09b87ed6808 100644 --- a/crates/evm/execution-types/src/chain.rs +++ b/crates/evm/execution-types/src/chain.rs @@ -11,7 +11,7 @@ use reth_primitives::{ TransactionSignedEcRecovered, }; use reth_primitives_traits::NodePrimitives; -use reth_trie::updates::TrieUpdates; +use reth_trie_common::updates::TrieUpdates; use revm::db::BundleState; /// A chain of blocks and their final state. @@ -513,16 +513,14 @@ pub enum ChainSplit { /// Bincode-compatible [`Chain`] serde implementation. #[cfg(all(feature = "serde", feature = "serde-bincode-compat"))] pub(super) mod serde_bincode_compat { - use std::collections::BTreeMap; - + use crate::ExecutionOutcome; use alloc::borrow::Cow; use alloy_primitives::BlockNumber; use reth_primitives::serde_bincode_compat::SealedBlockWithSenders; - use reth_trie::serde_bincode_compat::updates::TrieUpdates; + use reth_trie_common::serde_bincode_compat::updates::TrieUpdates; use serde::{ser::SerializeMap, Deserialize, Deserializer, Serialize, Serializer}; use serde_with::{DeserializeAs, SerializeAs}; - - use crate::ExecutionOutcome; + use std::collections::BTreeMap; /// Bincode-compatible [`super::Chain`] serde implementation. /// diff --git a/crates/primitives/Cargo.toml b/crates/primitives/Cargo.toml index ebfa26aef0a4..9787c9f3a6a9 100644 --- a/crates/primitives/Cargo.toml +++ b/crates/primitives/Cargo.toml @@ -158,11 +158,12 @@ test-utils = [ "arbitrary", ] serde-bincode-compat = [ + "serde_with", + "alloy-eips/serde-bincode-compat", "alloy-consensus/serde-bincode-compat", "op-alloy-consensus?/serde-bincode-compat", "reth-primitives-traits/serde-bincode-compat", - "serde_with", - "alloy-eips/serde-bincode-compat", + "reth-trie-common/serde-bincode-compat", ] [[bench]] diff --git a/crates/revm/Cargo.toml b/crates/revm/Cargo.toml index 4bc78b7b0562..95def23a4432 100644 --- a/crates/revm/Cargo.toml +++ b/crates/revm/Cargo.toml @@ -55,9 +55,9 @@ test-utils = [ ] serde = [ "revm/serde", - "reth-trie?/serde", "alloy-eips/serde", "alloy-primitives/serde", "alloy-consensus/serde", "reth-primitives-traits/serde", + "reth-trie?/serde", ] diff --git a/crates/storage/provider/Cargo.toml b/crates/storage/provider/Cargo.toml index 86f2f3a51b91..2875b91149c5 100644 --- a/crates/storage/provider/Cargo.toml +++ b/crates/storage/provider/Cargo.toml @@ -98,21 +98,21 @@ optimism = [ "revm/optimism", ] serde = [ - "reth-execution-types/serde", - "reth-trie-db/serde", - "reth-trie/serde", - "alloy-consensus/serde", - "alloy-eips/serde", - "alloy-primitives/serde", - "alloy-rpc-types-engine/serde", "dashmap/serde", "notify/serde", "parking_lot/serde", "rand/serde", + "alloy-primitives/serde", + "alloy-consensus/serde", + "alloy-eips/serde", + "alloy-rpc-types-engine/serde", "revm/serde", "reth-codecs/serde", "reth-optimism-primitives?/serde", "reth-primitives-traits/serde", + "reth-execution-types/serde", + "reth-trie-db/serde", + "reth-trie/serde", ] test-utils = [ "reth-db/test-utils", diff --git a/crates/trie/common/Cargo.toml b/crates/trie/common/Cargo.toml index 47c09d2a7c0e..29993ab13dd1 100644 --- a/crates/trie/common/Cargo.toml +++ b/crates/trie/common/Cargo.toml @@ -31,6 +31,9 @@ nybbles = { workspace = true, features = ["rlp"] } # `serde` feature serde = { workspace = true, optional = true } +# `serde-bincode-compat` feature +serde_with = { workspace = true, optional = true } + # `test-utils` feature hash-db = { version = "=0.15.2", optional = true } plain_hasher = { version = "0.2", optional = true } @@ -46,6 +49,7 @@ arbitrary = { workspace = true, features = ["derive"] } proptest.workspace = true proptest-arbitrary-interop.workspace = true criterion.workspace = true +bincode.workspace = true [features] serde = [ @@ -59,6 +63,11 @@ serde = [ "reth-primitives-traits/serde", "reth-codecs/serde" ] +serde-bincode-compat = [ + "serde_with", + "reth-primitives-traits/serde-bincode-compat", + "alloy-consensus/serde-bincode-compat" +] test-utils = [ "dep:plain_hasher", "dep:hash-db", diff --git a/crates/trie/common/src/lib.rs b/crates/trie/common/src/lib.rs index 6f3cbf3eeae7..04b817aab8f2 100644 --- a/crates/trie/common/src/lib.rs +++ b/crates/trie/common/src/lib.rs @@ -37,5 +37,19 @@ pub use proofs::*; pub mod root; +/// Buffer for trie updates. +pub mod updates; + +/// Bincode-compatible serde implementations for trie types. +/// +/// `bincode` crate allows for more efficient serialization of trie types, because it allows +/// non-string map keys. +/// +/// Read more: +#[cfg(all(feature = "serde", feature = "serde-bincode-compat"))] +pub mod serde_bincode_compat { + pub use super::updates::serde_bincode_compat as updates; +} + /// Re-export pub use alloy_trie::{nodes::*, proof, BranchNodeCompact, HashBuilder, TrieMask, EMPTY_ROOT_HASH}; diff --git a/crates/trie/trie/src/updates.rs b/crates/trie/common/src/updates.rs similarity index 93% rename from crates/trie/trie/src/updates.rs rename to crates/trie/common/src/updates.rs index e7bc490647cc..76aa37a47785 100644 --- a/crates/trie/trie/src/updates.rs +++ b/crates/trie/common/src/updates.rs @@ -1,4 +1,4 @@ -use crate::{walker::TrieWalker, BranchNodeCompact, HashBuilder, Nibbles}; +use crate::{BranchNodeCompact, HashBuilder, Nibbles}; use alloy_primitives::B256; use std::collections::{HashMap, HashSet}; @@ -78,20 +78,19 @@ impl TrieUpdates { } /// Finalize state trie updates. - pub fn finalize( + pub fn finalize( &mut self, - walker: TrieWalker, hash_builder: HashBuilder, + removed_keys: HashSet, destroyed_accounts: HashSet, ) { - // Retrieve deleted keys from trie walker. - let (_, removed_node_keys) = walker.split(); - self.removed_nodes.extend(exclude_empty(removed_node_keys)); - // Retrieve updated nodes from hash builder. let (_, updated_nodes) = hash_builder.split(); self.account_nodes.extend(exclude_empty_from_pair(updated_nodes)); + // Add deleted node paths. + self.removed_nodes.extend(exclude_empty(removed_keys)); + // Add deleted storage tries for destroyed accounts. for destroyed in destroyed_accounts { self.storage_tries.entry(destroyed).or_default().set_deleted(true); @@ -201,14 +200,13 @@ impl StorageTrieUpdates { } /// Finalize storage trie updates for by taking updates from walker and hash builder. - pub fn finalize(&mut self, walker: TrieWalker, hash_builder: HashBuilder) { - // Retrieve deleted keys from trie walker. - let (_, removed_keys) = walker.split(); - self.removed_nodes.extend(exclude_empty(removed_keys)); - + pub fn finalize(&mut self, hash_builder: HashBuilder, removed_keys: HashSet) { // Retrieve updated nodes from hash builder. let (_, updated_nodes) = hash_builder.split(); self.storage_nodes.extend(exclude_empty_from_pair(updated_nodes)); + + // Add deleted node paths. + self.removed_nodes.extend(exclude_empty(removed_keys)); } /// Convert storage trie updates into [`StorageTrieUpdatesSorted`]. @@ -229,10 +227,9 @@ impl StorageTrieUpdates { /// This also sorts the set before serializing. #[cfg(feature = "serde")] mod serde_nibbles_set { - use std::collections::HashSet; - - use reth_trie_common::Nibbles; + use crate::Nibbles; use serde::{de::Error, Deserialize, Deserializer, Serialize, Serializer}; + use std::collections::HashSet; pub(super) fn serialize(map: &HashSet, serializer: S) -> Result where @@ -266,15 +263,14 @@ mod serde_nibbles_set { /// This also sorts the map's keys before encoding and serializing. #[cfg(feature = "serde")] mod serde_nibbles_map { - use std::{collections::HashMap, marker::PhantomData}; - + use crate::Nibbles; use alloy_primitives::hex; - use reth_trie_common::Nibbles; use serde::{ de::{Error, MapAccess, Visitor}, ser::SerializeMap, Deserialize, Deserializer, Serialize, Serializer, }; + use std::{collections::HashMap, marker::PhantomData}; pub(super) fn serialize( map: &HashMap, @@ -340,9 +336,13 @@ mod serde_nibbles_map { /// Sorted trie updates used for lookups and insertions. #[derive(PartialEq, Eq, Clone, Default, Debug)] pub struct TrieUpdatesSorted { - pub(crate) account_nodes: Vec<(Nibbles, BranchNodeCompact)>, - pub(crate) removed_nodes: HashSet, - pub(crate) storage_tries: HashMap, + /// Sorted collection of updated state nodes with corresponding paths. + pub account_nodes: Vec<(Nibbles, BranchNodeCompact)>, + /// The set of removed state node keys. + pub removed_nodes: HashSet, + /// Storage tries storage stored by hashed address of the account + /// the trie belongs to. + pub storage_tries: HashMap, } impl TrieUpdatesSorted { @@ -365,9 +365,12 @@ impl TrieUpdatesSorted { /// Sorted trie updates used for lookups and insertions. #[derive(PartialEq, Eq, Clone, Default, Debug)] pub struct StorageTrieUpdatesSorted { - pub(crate) is_deleted: bool, - pub(crate) storage_nodes: Vec<(Nibbles, BranchNodeCompact)>, - pub(crate) removed_nodes: HashSet, + /// Flag indicating whether the trie has been deleted/wiped. + pub is_deleted: bool, + /// Sorted collection of updated storage nodes with corresponding paths. + pub storage_nodes: Vec<(Nibbles, BranchNodeCompact)>, + /// The set of removed storage node keys. + pub removed_nodes: HashSet, } impl StorageTrieUpdatesSorted { @@ -402,21 +405,20 @@ fn exclude_empty_from_pair( /// Bincode-compatible trie updates type serde implementations. #[cfg(all(feature = "serde", feature = "serde-bincode-compat"))] pub mod serde_bincode_compat { + use crate::{BranchNodeCompact, Nibbles}; + use alloy_primitives::B256; + use serde::{Deserialize, Deserializer, Serialize, Serializer}; + use serde_with::{DeserializeAs, SerializeAs}; use std::{ borrow::Cow, collections::{HashMap, HashSet}, }; - use alloy_primitives::B256; - use reth_trie_common::{BranchNodeCompact, Nibbles}; - use serde::{Deserialize, Deserializer, Serialize, Serializer}; - use serde_with::{DeserializeAs, SerializeAs}; - /// Bincode-compatible [`super::TrieUpdates`] serde implementation. /// /// Intended to use with the [`serde_with::serde_as`] macro in the following way: /// ```rust - /// use reth_trie::{serde_bincode_compat, updates::TrieUpdates}; + /// use reth_trie_common::{serde_bincode_compat, updates::TrieUpdates}; /// use serde::{Deserialize, Serialize}; /// use serde_with::serde_as; /// @@ -480,7 +482,7 @@ pub mod serde_bincode_compat { /// /// Intended to use with the [`serde_with::serde_as`] macro in the following way: /// ```rust - /// use reth_trie::{serde_bincode_compat, updates::StorageTrieUpdates}; + /// use reth_trie_common::{serde_bincode_compat, updates::StorageTrieUpdates}; /// use serde::{Deserialize, Serialize}; /// use serde_with::serde_as; /// @@ -541,12 +543,12 @@ pub mod serde_bincode_compat { #[cfg(test)] mod tests { - use crate::updates::StorageTrieUpdates; - - use super::super::{serde_bincode_compat, TrieUpdates}; - + use crate::{ + serde_bincode_compat, + updates::{StorageTrieUpdates, TrieUpdates}, + BranchNodeCompact, Nibbles, + }; use alloy_primitives::B256; - use reth_trie_common::{BranchNodeCompact, Nibbles}; use serde::{Deserialize, Serialize}; use serde_with::serde_as; @@ -555,7 +557,7 @@ pub mod serde_bincode_compat { #[serde_as] #[derive(Debug, PartialEq, Eq, Serialize, Deserialize)] struct Data { - #[serde_as(as = "serde_bincode_compat::TrieUpdates")] + #[serde_as(as = "serde_bincode_compat::updates::TrieUpdates")] trie_updates: TrieUpdates, } @@ -588,7 +590,7 @@ pub mod serde_bincode_compat { #[serde_as] #[derive(Debug, PartialEq, Eq, Serialize, Deserialize)] struct Data { - #[serde_as(as = "serde_bincode_compat::StorageTrieUpdates")] + #[serde_as(as = "serde_bincode_compat::updates::StorageTrieUpdates")] trie_updates: StorageTrieUpdates, } diff --git a/crates/trie/db/Cargo.toml b/crates/trie/db/Cargo.toml index 74f9f98f1bf7..2fbdf1d57561 100644 --- a/crates/trie/db/Cargo.toml +++ b/crates/trie/db/Cargo.toml @@ -67,16 +67,17 @@ similar-asserts.workspace = true metrics = ["reth-metrics", "reth-trie/metrics", "dep:metrics"] serde = [ "dep:serde", - "reth-provider/serde", - "reth-trie/serde", - "reth-trie-common/serde", + "similar-asserts/serde", + "revm/serde", "alloy-consensus/serde", "alloy-primitives/serde", - "revm/serde", - "similar-asserts/serde" + "reth-trie/serde", + "reth-trie-common/serde", + "reth-provider/serde", ] test-utils = [ "triehash", + "revm/test-utils", "reth-trie-common/test-utils", "reth-chainspec/test-utils", "reth-primitives/test-utils", @@ -84,5 +85,4 @@ test-utils = [ "reth-db-api/test-utils", "reth-provider/test-utils", "reth-trie/test-utils", - "revm/test-utils" ] diff --git a/crates/trie/parallel/src/root.rs b/crates/trie/parallel/src/root.rs index 7a316d8b15fb..b4e300c7290e 100644 --- a/crates/trie/parallel/src/root.rs +++ b/crates/trie/parallel/src/root.rs @@ -193,11 +193,8 @@ where let root = hash_builder.root(); - trie_updates.finalize( - account_node_iter.walker, - hash_builder, - prefix_sets.destroyed_accounts, - ); + let removed_keys = account_node_iter.walker.take_removed_keys(); + trie_updates.finalize(hash_builder, removed_keys, prefix_sets.destroyed_accounts); let stats = tracker.finish(); diff --git a/crates/trie/sparse/benches/root.rs b/crates/trie/sparse/benches/root.rs index 30ce566fb5f6..d8d210c1b19d 100644 --- a/crates/trie/sparse/benches/root.rs +++ b/crates/trie/sparse/benches/root.rs @@ -146,7 +146,7 @@ pub fn calculate_root_from_leaves_repeated(c: &mut Criterion) { hb.root(); if storage_updates.peek().is_some() { - trie_updates.finalize(node_iter.walker, hb); + trie_updates.finalize(hb, node_iter.walker.take_removed_keys()); } } }, diff --git a/crates/trie/trie/Cargo.toml b/crates/trie/trie/Cargo.toml index fd4d80ce7e3a..c1c3ae4dd876 100644 --- a/crates/trie/trie/Cargo.toml +++ b/crates/trie/trie/Cargo.toml @@ -42,12 +42,6 @@ metrics = { workspace = true, optional = true } # `test-utils` feature triehash = { version = "0.8", optional = true } -# `serde` feature -serde = { workspace = true, optional = true } - -# `serde-bincode-compat` feature -serde_with = { workspace = true, optional = true } - [dev-dependencies] # reth reth-primitives = { workspace = true, features = ["test-utils", "arbitrary"] } @@ -61,28 +55,21 @@ proptest.workspace = true proptest-arbitrary-interop.workspace = true serde_json.workspace = true criterion.workspace = true -bincode.workspace = true [features] metrics = ["reth-metrics", "dep:metrics"] serde = [ - "dep:serde", - "alloy-consensus/serde", "alloy-primitives/serde", - "revm/serde", + "alloy-consensus/serde", "alloy-trie/serde", + "revm/serde", "reth-trie-common/serde" ] -serde-bincode-compat = [ - "serde_with", - "reth-primitives/serde-bincode-compat", - "alloy-consensus/serde-bincode-compat" -] test-utils = [ "triehash", - "reth-trie-common/test-utils", - "reth-primitives/test-utils", "revm/test-utils", + "reth-primitives/test-utils", + "reth-trie-common/test-utils", "reth-stages-types/test-utils" ] diff --git a/crates/trie/trie/src/lib.rs b/crates/trie/trie/src/lib.rs index 73a08d3fc442..335711b8d88e 100644 --- a/crates/trie/trie/src/lib.rs +++ b/crates/trie/trie/src/lib.rs @@ -50,9 +50,6 @@ pub mod witness; mod trie; pub use trie::{StateRoot, StorageRoot}; -/// Buffer for trie updates. -pub mod updates; - /// Utilities for state root checkpoint progress. mod progress; pub use progress::{IntermediateStateRootState, StateRootProgress}; @@ -63,17 +60,6 @@ pub mod stats; // re-export for convenience pub use reth_trie_common::*; -/// Bincode-compatible serde implementations for trie types. -/// -/// `bincode` crate allows for more efficient serialization of trie types, because it allows -/// non-string map keys. -/// -/// Read more: -#[cfg(all(feature = "serde", feature = "serde-bincode-compat"))] -pub mod serde_bincode_compat { - pub use super::updates::serde_bincode_compat as updates; -} - /// Trie calculation metrics. #[cfg(feature = "metrics")] pub mod metrics; diff --git a/crates/trie/trie/src/trie.rs b/crates/trie/trie/src/trie.rs index 74faf7bbc60f..28517b23e90f 100644 --- a/crates/trie/trie/src/trie.rs +++ b/crates/trie/trie/src/trie.rs @@ -258,11 +258,8 @@ where let root = hash_builder.root(); - trie_updates.finalize( - account_node_iter.walker, - hash_builder, - self.prefix_sets.destroyed_accounts, - ); + let removed_keys = account_node_iter.walker.take_removed_keys(); + trie_updates.finalize(hash_builder, removed_keys, self.prefix_sets.destroyed_accounts); let stats = tracker.finish(); @@ -434,7 +431,8 @@ where let root = hash_builder.root(); let mut trie_updates = StorageTrieUpdates::default(); - trie_updates.finalize(storage_node_iter.walker, hash_builder); + let removed_keys = storage_node_iter.walker.take_removed_keys(); + trie_updates.finalize(hash_builder, removed_keys); let stats = tracker.finish(); diff --git a/crates/trie/trie/src/walker.rs b/crates/trie/trie/src/walker.rs index 774fa64a0efe..146dbc213e5a 100644 --- a/crates/trie/trie/src/walker.rs +++ b/crates/trie/trie/src/walker.rs @@ -58,8 +58,13 @@ impl TrieWalker { /// Split the walker into stack and trie updates. pub fn split(mut self) -> (Vec, HashSet) { - let keys = self.removed_keys.take(); - (self.stack, keys.unwrap_or_default()) + let keys = self.take_removed_keys(); + (self.stack, keys) + } + + /// Take removed keys from the walker. + pub fn take_removed_keys(&mut self) -> HashSet { + self.removed_keys.take().unwrap_or_default() } /// Prints the current stack of trie nodes.