diff --git a/Cargo.lock b/Cargo.lock index cf33ea0..1ec3b40 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -240,9 +240,9 @@ checksum = "96d30a06541fbafbc7f82ed10c06164cfbd2c401138f6addd8404629c4b16711" [[package]] name = "async-trait" -version = "0.1.73" +version = "0.1.74" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bc00ceb34980c03614e35a3a4e218276a0a824e911d07651cd0d858a51e8c0f0" +checksum = "a66537f1bb974b254c98ed142ff995236e81b9d0fe4db0575f46612cb15eb0f9" dependencies = [ "proc-macro2 1.0.69", "quote 1.0.33", @@ -788,9 +788,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.4.6" +version = "4.4.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d04704f56c2cde07f43e8e2c154b43f216dc5c92fc98ada720177362f953b956" +checksum = "ac495e00dcec98c83465d5ad66c5c4fabd652fd6686e7c6269b117e729a6f17b" dependencies = [ "clap_builder", "clap_derive", @@ -798,9 +798,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.4.6" +version = "4.4.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0e231faeaca65ebd1ea3c737966bf858971cd38c3849107aa3ea7de90a804e45" +checksum = "c77ed9a32a62e6ca27175d00d29d05ca32e396ea1eb5fb01d8256b669cec7663" dependencies = [ "anstream", "anstyle", @@ -810,9 +810,9 @@ dependencies = [ [[package]] name = "clap_derive" -version = "4.4.2" +version = "4.4.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0862016ff20d69b84ef8247369fabf5c008a7417002411897d40ee1f4532b873" +checksum = "cf9804afaaf59a91e75b022a30fb7229a7901f60c755489cc61c9b423b836442" dependencies = [ "heck 0.4.1", "proc-macro2 1.0.69", @@ -822,9 +822,9 @@ dependencies = [ [[package]] name = "clap_lex" -version = "0.5.1" +version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cd7cc57abe963c6d3b9d8be5b06ba7c8957a930305ca90304f24ef040aa6f961" +checksum = "702fc72eb24e5a1e48ce58027a675bc24edd52096d5397d4aea7c6dd9eca0bd1" [[package]] name = "cloudabi" @@ -4375,9 +4375,9 @@ dependencies = [ [[package]] name = "serde" -version = "1.0.188" +version = "1.0.189" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf9e0fcba69a370eed61bcf2b728575f726b50b55cba78064753d708ddc7549e" +checksum = "8e422a44e74ad4001bdc8eede9a4570ab52f71190e9c076d14369f38b9200537" dependencies = [ "serde_derive", ] @@ -4394,9 +4394,9 @@ dependencies = [ [[package]] name = "serde_derive" -version = "1.0.188" +version = "1.0.189" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4eca7ac642d82aa35b60049a6eccb4be6be75e599bd2e9adb5f875a737654af2" +checksum = "1e48d1f918009ce3145511378cf68d613e3b3d9137d67272562080d68a2b32d5" dependencies = [ "proc-macro2 1.0.69", "quote 1.0.33", @@ -4631,11 +4631,27 @@ version = "0.1.0" dependencies = [ "async-trait", "blake2 0.10.6", - "clap 4.4.6", + "clap 4.4.7", "ethers", "eyre", "hex", "indexmap 2.0.2", + "serde", + "serde_json", + "state-reconstruct-fetcher", + "tokio", + "tracing", + "tracing-subscriber", + "zksync_merkle_tree", +] + +[[package]] +name = "state-reconstruct-fetcher" +version = "0.1.0" +dependencies = [ + "ethers", + "eyre", + "indexmap 2.0.2", "rand 0.8.5", "serde", "serde_json", @@ -4643,8 +4659,6 @@ dependencies = [ "thiserror", "tokio", "tracing", - "tracing-subscriber", - "zksync_merkle_tree", ] [[package]] @@ -4847,18 +4861,18 @@ dependencies = [ [[package]] name = "thiserror" -version = "1.0.49" +version = "1.0.50" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1177e8c6d7ede7afde3585fd2513e611227efd6481bd78d2e82ba1ce16557ed4" +checksum = "f9a7210f5c9a7156bb50aa36aed4c95afb51df0df00713949448cf9e97d382d2" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "1.0.49" +version = "1.0.50" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "10712f02019e9288794769fba95cd6847df9874d49d871d062172f9dd41bc4cc" +checksum = "266b2e40bc00e5a6c09c3584011e08b06f123c00362c92b975ba9843aaaa14b8" dependencies = [ "proc-macro2 1.0.69", "quote 1.0.33", @@ -5033,11 +5047,10 @@ checksum = "b6bc1c9ce2b5135ac7f93c72918fc37feb872bdc6a5533a8b85eb4b86bfdae52" [[package]] name = "tracing" -version = "0.1.37" +version = "0.1.40" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ce8c33a8d48bd45d624a6e523445fd21ec13d3653cd51f681abf67418f54eb8" +checksum = "c3523ab5a71916ccf420eebdf5521fcef02141234bbc0b8a49f2fdc4544364ef" dependencies = [ - "cfg-if 1.0.0", "pin-project-lite", "tracing-attributes", "tracing-core", @@ -5045,9 +5058,9 @@ dependencies = [ [[package]] name = "tracing-attributes" -version = "0.1.26" +version = "0.1.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f4f31f56159e98206da9efd823404b79b6ef3143b4a7ab76e67b1751b25a4ab" +checksum = "34704c8d6ebcbc939824180af020566b01a7c01f80641264eba0999f6c2b6be7" dependencies = [ "proc-macro2 1.0.69", "quote 1.0.33", @@ -5056,9 +5069,9 @@ dependencies = [ [[package]] name = "tracing-core" -version = "0.1.31" +version = "0.1.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0955b8137a1df6f1a2e9a37d8a6656291ff0297c1a97c24e0d8425fe2312f79a" +checksum = "c06d3da6113f116aaee68e4d601191614c9053067f9ab7f6edbcb161237daa54" dependencies = [ "once_cell", "valuable", diff --git a/Cargo.toml b/Cargo.toml index 19a6af1..2be4481 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -4,20 +4,21 @@ version = "0.1.0" edition = "2021" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html +[workspace] +members = ["state-reconstruct-fetcher"] + [dependencies] -async-trait = "0.1.73" +async-trait = "0.1.74" blake2 = "0.10.6" -clap = { version = "4.4.0", features = ["derive", "env"] } -ethers = "1" +clap = { version = "4.4.7", features = ["derive", "env"] } +ethers = "1.0.2" eyre = "0.6.8" hex = "0.4.3" -indexmap = { version = "2.0.1", features = ["serde"] } -rand = "0.8.0" -serde = { version = "1.0.188", features = ["derive"] } +indexmap = { version = "2.0.2" } +serde = { version = "1.0.189", features = ["derive"] } serde_json = { version = "1.0.107", features = ["std"] } -serde_json_any_key = "2.0.0" -thiserror = "1.0" -tokio = { version = "1.32.0", features = ["macros", "signal"] } -tracing = "0.1.37" +state-reconstruct-fetcher = { path = "./state-reconstruct-fetcher" } +tokio = { version = "1.33.0", features = ["macros"] } +tracing = "0.1.40" tracing-subscriber = "0.3.17" zksync_merkle_tree = { git = "https://github.com/matter-labs/zksync-era.git" } diff --git a/src/cli.rs b/src/cli.rs index 52a17f8..a049726 100644 --- a/src/cli.rs +++ b/src/cli.rs @@ -1,6 +1,5 @@ use clap::{Args, Parser, Subcommand, ValueEnum}; - -use crate::constants::ethereum; +use state_reconstruct_fetcher::constants::ethereum; #[derive(Args)] pub struct L1FetcherOptions { diff --git a/src/main.rs b/src/main.rs index ff6c02e..024152e 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,14 +1,8 @@ -#![feature(array_chunks)] -#![feature(iter_next_chunk)] #![warn(clippy::pedantic)] #![allow(clippy::module_name_repetitions)] mod cli; -mod constants; -mod l1_fetcher; mod processor; -mod snapshot; -mod types; mod util; use std::{ @@ -21,20 +15,22 @@ use std::{ use clap::Parser; use cli::{Cli, Command, ReconstructSource}; -use constants::storage; use eyre::Result; -use snapshot::StateSnapshot; +use state_reconstruct_fetcher::{ + constants::storage, + l1_fetcher::{L1Fetcher, L1FetcherOptions}, + snapshot::StateSnapshot, + types::CommitBlockInfoV1, +}; use tokio::sync::{mpsc, Mutex}; use tracing_subscriber::{filter::LevelFilter, EnvFilter}; use crate::{ - l1_fetcher::L1Fetcher, processor::{ json::JsonSerializationProcessor, tree::{query_tree::QueryTree, TreeProcessor}, Processor, }, - types::CommitBlockInfoV1, util::json, }; @@ -71,8 +67,15 @@ async fn main() -> Result<()> { match source { ReconstructSource::L1 { l1_fetcher_options } => { let snapshot = Arc::new(Mutex::new(StateSnapshot::default())); - - let fetcher = L1Fetcher::new(l1_fetcher_options, Some(snapshot.clone()))?; + let fetcher_options = L1FetcherOptions { + http_url: l1_fetcher_options.http_url, + start_block: l1_fetcher_options.start_block, + block_step: l1_fetcher_options.block_step, + block_count: l1_fetcher_options.block_count, + disable_polling: l1_fetcher_options.disable_polling, + }; + + let fetcher = L1Fetcher::new(fetcher_options, Some(snapshot.clone()))?; let processor = TreeProcessor::new(db_path, snapshot.clone()).await?; let (tx, rx) = mpsc::channel::(5); @@ -109,7 +112,15 @@ async fn main() -> Result<()> { l1_fetcher_options, file, } => { - let fetcher = L1Fetcher::new(l1_fetcher_options, None)?; + let fetcher_options = L1FetcherOptions { + http_url: l1_fetcher_options.http_url, + start_block: l1_fetcher_options.start_block, + block_step: l1_fetcher_options.block_step, + block_count: l1_fetcher_options.block_count, + disable_polling: l1_fetcher_options.disable_polling, + }; + + let fetcher = L1Fetcher::new(fetcher_options, None)?; let processor = JsonSerializationProcessor::new(Path::new(&file))?; let (tx, rx) = mpsc::channel::(5); diff --git a/src/processor/json/mod.rs b/src/processor/json/mod.rs index b7a77b4..34594c7 100644 --- a/src/processor/json/mod.rs +++ b/src/processor/json/mod.rs @@ -4,10 +4,10 @@ use async_trait::async_trait; use eyre::Result; use serde::ser::{SerializeSeq, Serializer}; use serde_json; +use state_reconstruct_fetcher::types::CommitBlockInfoV1; use tokio::sync::mpsc; use super::Processor; -use crate::types::CommitBlockInfoV1; pub struct JsonSerializationProcessor { serializer: serde_json::Serializer, diff --git a/src/processor/mod.rs b/src/processor/mod.rs index 4e014c4..316d05d 100644 --- a/src/processor/mod.rs +++ b/src/processor/mod.rs @@ -1,8 +1,7 @@ use async_trait::async_trait; +use state_reconstruct_fetcher::types::CommitBlockInfoV1; use tokio::sync::mpsc; -use crate::types::CommitBlockInfoV1; - pub mod json; pub mod tree; diff --git a/src/processor/tree/mod.rs b/src/processor/tree/mod.rs index 902aede..e5f89cf 100644 --- a/src/processor/tree/mod.rs +++ b/src/processor/tree/mod.rs @@ -6,13 +6,13 @@ use std::{io, path::PathBuf, sync::Arc}; use async_trait::async_trait; use ethers::types::H256; use eyre::Result; +use state_reconstruct_fetcher::{ + constants::storage::STATE_FILE_NAME, snapshot::StateSnapshot, types::CommitBlockInfoV1, +}; use tokio::sync::{mpsc, Mutex}; use self::tree_wrapper::TreeWrapper; use super::Processor; -use crate::{ - constants::storage::STATE_FILE_NAME, snapshot::StateSnapshot, types::CommitBlockInfoV1, -}; pub type RootHash = H256; diff --git a/src/processor/tree/tree_wrapper.rs b/src/processor/tree/tree_wrapper.rs index 26f5b54..845e7fd 100644 --- a/src/processor/tree/tree_wrapper.rs +++ b/src/processor/tree/tree_wrapper.rs @@ -4,10 +4,10 @@ use blake2::{Blake2s256, Digest}; use ethers::types::{Address, H256, U256}; use eyre::Result; use indexmap::IndexSet; +use state_reconstruct_fetcher::{constants::storage::INITAL_STATE_PATH, types::CommitBlockInfoV1}; use zksync_merkle_tree::{Database, MerkleTree, RocksDBWrapper}; use super::RootHash; -use crate::{constants::storage::INITAL_STATE_PATH, CommitBlockInfoV1}; pub struct TreeWrapper<'a> { tree: MerkleTree<'a, RocksDBWrapper>, diff --git a/state-reconstruct-fetcher/Cargo.toml b/state-reconstruct-fetcher/Cargo.toml new file mode 100644 index 0000000..14bc0c1 --- /dev/null +++ b/state-reconstruct-fetcher/Cargo.toml @@ -0,0 +1,16 @@ +[package] +name = "state-reconstruct-fetcher" +version = "0.1.0" +edition = "2021" + +[dependencies] +ethers = "1.0.2" +eyre = "0.6.8" +indexmap = { version = "2.0.2", features = ["serde"] } +rand = "0.8.5" +serde = { version = "1.0.189", features = ["derive"] } +serde_json = { version = "1.0.107", features = ["std"] } +serde_json_any_key = "2.0.0" +thiserror = "1.0.50" +tokio = { version = "1.33.0", features = ["signal"] } +tracing = "0.1.40" diff --git a/src/constants.rs b/state-reconstruct-fetcher/src/constants.rs similarity index 100% rename from src/constants.rs rename to state-reconstruct-fetcher/src/constants.rs diff --git a/src/l1_fetcher.rs b/state-reconstruct-fetcher/src/l1_fetcher.rs similarity index 96% rename from src/l1_fetcher.rs rename to state-reconstruct-fetcher/src/l1_fetcher.rs index dbdcea6..c0183b2 100644 --- a/src/l1_fetcher.rs +++ b/state-reconstruct-fetcher/src/l1_fetcher.rs @@ -14,7 +14,6 @@ use tokio::{ }; use crate::{ - cli::L1FetcherOptions, constants::ethereum::{BLOCK_STEP, GENESIS_BLOCK, ZK_SYNC_ADDR}, snapshot::StateSnapshot, types::{CommitBlockInfoV1, ParseError}, @@ -37,6 +36,19 @@ pub enum L1FetchError { GetTx, } +pub struct L1FetcherOptions { + /// The Ethereum JSON-RPC HTTP URL to use. + pub http_url: String, + /// Ethereum block number to start state import from. + pub start_block: u64, + /// The number of blocks to filter & process in one step over. + pub block_step: u64, + /// The number of blocks to process from Ethereum. + pub block_count: Option, + /// If present, don't poll for new blocks after reaching the end. + pub disable_polling: bool, +} + #[derive(Default)] struct L1Metrics { // Metrics variables. diff --git a/state-reconstruct-fetcher/src/lib.rs b/state-reconstruct-fetcher/src/lib.rs new file mode 100644 index 0000000..9efa939 --- /dev/null +++ b/state-reconstruct-fetcher/src/lib.rs @@ -0,0 +1,6 @@ +#![feature(array_chunks)] +#![feature(iter_next_chunk)] +pub mod constants; +pub mod l1_fetcher; +pub mod snapshot; +pub mod types; diff --git a/src/snapshot.rs b/state-reconstruct-fetcher/src/snapshot.rs similarity index 100% rename from src/snapshot.rs rename to state-reconstruct-fetcher/src/snapshot.rs diff --git a/src/types.rs b/state-reconstruct-fetcher/src/types.rs similarity index 100% rename from src/types.rs rename to state-reconstruct-fetcher/src/types.rs