ref: move storage to its own crate (#87)

* ref: move storage to its own crate

Cargo.toml

@@ -5,35 +5,33 @@ edition = "2021"
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [workspace]
-members = ["state-reconstruct-fetcher", "state-reconstruct-fetcher/blobscan-client"]
+members = [
+  "state-reconstruct-fetcher",
+  "state-reconstruct-fetcher/blobscan-client",
+  "state-reconstruct-storage",
+]
 
 [dependencies]
 async-trait = "0.1.74"
 bincode = "1"
 blake2 = "0.10.6"
 blobscan-client = { path = "./state-reconstruct-fetcher/blobscan-client" }
-bytes = "1.5"
 chrono = "0.4.31"
 clap = { version = "4.4.7", features = ["derive", "env"] }
 ethers = "1.0.2"
 eyre = "0.6.8"
-flate2 = "1.0.28"
 hex = "0.4.3"
 indexmap = { version = "2.0.2" }
 primitive-types = "0.12.2"
-prost = "0.12"
 rocksdb = "0.21"
 serde = { version = "1.0.189", features = ["derive"] }
 serde_json = { version = "1.0.107", features = ["std"] }
 state-reconstruct-fetcher = { path = "./state-reconstruct-fetcher" }
+state-reconstruct-storage = { path = "./state-reconstruct-storage" }
 thiserror = "1.0.50"
 tikv-jemallocator = "0.5"
 tokio = { version = "1.33.0", features = ["macros"] }
 tracing = "0.1.40"
 tracing-subscriber = "0.3.17"
-zkevm_opcode_defs = { git = "https://github.com/matter-labs/era-zkevm_opcode_defs.git" }
 zksync_merkle_tree = { git = "https://github.com/matter-labs/zksync-era.git" }
 zksync_storage = { git = "https://github.com/matter-labs/zksync-era.git" }
-
-[build-dependencies]
-prost-build = "0.12"

src/main.rs

@@ -86,7 +86,7 @@ async fn main() -> Result<()> {
                 ReconstructSource::L1 { l1_fetcher_options } => {
                     let fetcher_options = l1_fetcher_options.into();
                     let processor = TreeProcessor::new(db_path.clone()).await?;
-                    let fetcher = L1Fetcher::new(fetcher_options, Some(processor.get_snapshot()))?;
+                    let fetcher = L1Fetcher::new(fetcher_options, Some(processor.get_inner_db()))?;
                     let (tx, rx) = mpsc::channel::<CommitBlock>(5);
 
                     let processor_handle = tokio::spawn(async move {
@@ -160,7 +160,8 @@ async fn main() -> Result<()> {
             let processor = SnapshotBuilder::new(db_path);
 
             let mut fetcher_options: L1FetcherOptions = l1_fetcher_options.into();
-            if let Ok(Some(batch_number)) = processor.get_last_l1_batch_number() {
+            if let Ok(batch_number) = processor.get_latest_l1_batch_number() {
+                let batch_number = batch_number.as_u64();
                 if batch_number > ethereum::GENESIS_BLOCK {
                     tracing::info!(
                         "Found a preexisting snapshot db, continuing from L1 block: {batch_number}"

src/processor/snapshot/exporter.rs

@@ -1,20 +1,24 @@
 use std::path::{Path, PathBuf};
 
-use ethers::types::{U256, U64};
+use ethers::types::U256;
 use eyre::Result;
+use state_reconstruct_storage::{
+    snapshot::SnapshotDatabase,
+    snapshot_columns,
+    types::{
+        Proto, SnapshotFactoryDependencies, SnapshotFactoryDependency, SnapshotHeader,
+        SnapshotStorageLogsChunk, SnapshotStorageLogsChunkMetadata,
+    },
+    INDEX_TO_KEY_MAP,
+};
 
-use super::{
-    database::{self, SnapshotDB},
-    types::{SnapshotFactoryDependency, SnapshotHeader},
+use crate::processor::snapshot::{
     DEFAULT_DB_PATH, SNAPSHOT_FACTORY_DEPS_FILE_NAME_SUFFIX, SNAPSHOT_HEADER_FILE_NAME,
 };
-use crate::processor::snapshot::types::{
-    Proto, SnapshotFactoryDependencies, SnapshotStorageLogsChunk, SnapshotStorageLogsChunkMetadata,
-};
 
 pub struct SnapshotExporter {
     basedir: PathBuf,
-    database: SnapshotDB,
+    database: SnapshotDatabase,
 }
 
 impl SnapshotExporter {
@@ -24,20 +28,15 @@ impl SnapshotExporter {
             None => PathBuf::from(DEFAULT_DB_PATH),
         };
 
-        let database = SnapshotDB::new_read_only(db_path)?;
+        let database = SnapshotDatabase::new_read_only(db_path)?;
         Ok(Self {
             basedir: basedir.to_path_buf(),
             database,
         })
     }
 
     pub fn export_snapshot(&self, chunk_size: u64) -> Result<()> {
-        let l1_batch_number = U64::from(
-            self.database
-                .get_last_l1_batch_number()?
-                .expect("snapshot db contains no L1 batch number"),
-        );
-
+        let l1_batch_number = self.database.get_latest_l1_batch_number()?;
         let mut header = SnapshotHeader {
             l1_batch_number,
             ..Default::default()
@@ -61,7 +60,10 @@ impl SnapshotExporter {
     fn export_factory_deps(&self, header: &mut SnapshotHeader) -> Result<()> {
         tracing::info!("Exporting factory dependencies...");
 
-        let storage_logs = self.database.cf_handle(database::FACTORY_DEPS).unwrap();
+        let storage_logs = self
+            .database
+            .cf_handle(snapshot_columns::FACTORY_DEPS)
+            .unwrap();
         let mut iterator = self
             .database
             .iterator_cf(storage_logs, rocksdb::IteratorMode::Start);
@@ -93,7 +95,7 @@ impl SnapshotExporter {
         let num_logs = self.database.get_last_repeated_key_index()?;
         tracing::info!("Found {num_logs} logs.");
 
-        let index_to_key_map = self.database.cf_handle(database::INDEX_TO_KEY_MAP).unwrap();
+        let index_to_key_map = self.database.cf_handle(INDEX_TO_KEY_MAP).unwrap();
         let mut iterator = self
             .database
             .iterator_cf(index_to_key_map, rocksdb::IteratorMode::Start);

src/processor/snapshot/importer.rs

@@ -5,13 +5,14 @@ use std::{
 };
 
 use eyre::Result;
-use state_reconstruct_fetcher::{constants::storage::INNER_DB_NAME, database::InnerDB};
-use tokio::sync::Mutex;
-
-use super::{
+use state_reconstruct_fetcher::constants::storage::INNER_DB_NAME;
+use state_reconstruct_storage::{
+    reconstruction::ReconstructionDatabase,
     types::{Proto, SnapshotFactoryDependencies, SnapshotHeader, SnapshotStorageLogsChunk},
-    SNAPSHOT_FACTORY_DEPS_FILE_NAME_SUFFIX, SNAPSHOT_HEADER_FILE_NAME,
 };
+use tokio::sync::Mutex;
+
+use super::{SNAPSHOT_FACTORY_DEPS_FILE_NAME_SUFFIX, SNAPSHOT_HEADER_FILE_NAME};
 use crate::processor::tree::tree_wrapper::TreeWrapper;
 
 pub struct SnapshotImporter {
@@ -24,7 +25,7 @@ pub struct SnapshotImporter {
 impl SnapshotImporter {
     pub async fn new(directory: PathBuf, db_path: &Path) -> Result<Self> {
         let inner_db_path = db_path.join(INNER_DB_NAME);
-        let new_state = InnerDB::new(inner_db_path.clone())?;
+        let new_state = ReconstructionDatabase::new(inner_db_path.clone())?;
         let snapshot = Arc::new(Mutex::new(new_state));
         let tree = TreeWrapper::new(db_path, snapshot.clone(), true).await?;