Skip to content

Commit

Permalink
ref: separate db into two structs
Browse files Browse the repository at this point in the history
  • Loading branch information
zeapoz committed Apr 16, 2024
1 parent 5b97788 commit ea60992
Show file tree
Hide file tree
Showing 9 changed files with 392 additions and 295 deletions.
7 changes: 4 additions & 3 deletions src/processor/snapshot/exporter.rs
Original file line number Diff line number Diff line change
Expand Up @@ -3,12 +3,13 @@ use std::path::{Path, PathBuf};
use ethers::types::U256;
use eyre::Result;
use state_reconstruct_storage::{
snapshot::SnapshotDatabase,
snapshot_columns,
types::{
Proto, SnapshotFactoryDependencies, SnapshotFactoryDependency, SnapshotHeader,
SnapshotStorageLogsChunk, SnapshotStorageLogsChunkMetadata,
},
DBMode, InnerDB, INDEX_TO_KEY_MAP,
INDEX_TO_KEY_MAP,
};

use crate::processor::snapshot::{
Expand All @@ -17,7 +18,7 @@ use crate::processor::snapshot::{

pub struct SnapshotExporter {
basedir: PathBuf,
database: InnerDB,
database: SnapshotDatabase,
}

impl SnapshotExporter {
Expand All @@ -27,7 +28,7 @@ impl SnapshotExporter {
None => PathBuf::from(DEFAULT_DB_PATH),
};

let database = InnerDB::new_read_only(db_path, DBMode::Snapshot)?;
let database = SnapshotDatabase::new_read_only(db_path)?;
Ok(Self {
basedir: basedir.to_path_buf(),
database,
Expand Down
4 changes: 2 additions & 2 deletions src/processor/snapshot/importer.rs
Original file line number Diff line number Diff line change
Expand Up @@ -7,8 +7,8 @@ use std::{
use eyre::Result;
use state_reconstruct_fetcher::constants::storage::INNER_DB_NAME;
use state_reconstruct_storage::{
reconstruction::ReconstructionDatabase,
types::{Proto, SnapshotFactoryDependencies, SnapshotHeader, SnapshotStorageLogsChunk},
DBMode, InnerDB,
};
use tokio::sync::Mutex;

Expand All @@ -25,7 +25,7 @@ pub struct SnapshotImporter {
impl SnapshotImporter {
pub async fn new(directory: PathBuf, db_path: &Path) -> Result<Self> {
let inner_db_path = db_path.join(INNER_DB_NAME);
let new_state = InnerDB::new(inner_db_path.clone(), DBMode::Reconstruction)?;
let new_state = ReconstructionDatabase::new(inner_db_path.clone())?;
let snapshot = Arc::new(Mutex::new(new_state));
let tree = TreeWrapper::new(db_path, snapshot.clone(), true).await?;

Expand Down
13 changes: 7 additions & 6 deletions src/processor/snapshot/mod.rs
Original file line number Diff line number Diff line change
@@ -1,5 +1,7 @@
use std::{fs, path::PathBuf, str::FromStr};

use state_reconstruct_storage::snapshot::SnapshotDatabase;

pub mod exporter;
pub mod importer;

Expand All @@ -14,7 +16,6 @@ use state_reconstruct_fetcher::{
use state_reconstruct_storage::{
bytecode,
types::{MiniblockNumber, SnapshotFactoryDependency, SnapshotStorageLog},
DBMode, InnerDB,
};
use tokio::sync::mpsc;

Expand All @@ -25,7 +26,7 @@ pub const SNAPSHOT_HEADER_FILE_NAME: &str = "snapshot-header.json";
pub const SNAPSHOT_FACTORY_DEPS_FILE_NAME_SUFFIX: &str = "factory_deps.proto.gzip";

pub struct SnapshotBuilder {
database: InnerDB,
database: SnapshotDatabase,
}

impl SnapshotBuilder {
Expand All @@ -35,7 +36,7 @@ impl SnapshotBuilder {
None => PathBuf::from(DEFAULT_DB_PATH),
};

let mut database = InnerDB::new(db_path, DBMode::Snapshot).unwrap();
let mut database = SnapshotDatabase::new(db_path).unwrap();

let idx = database
.get_last_repeated_key_index()
Expand Down Expand Up @@ -128,7 +129,7 @@ impl Processor for SnapshotBuilder {

// TODO: Can this be made somewhat generic?
/// Attempts to reconstruct the genesis state from a CSV file.
fn reconstruct_genesis_state(database: &mut InnerDB, path: &str) -> Result<()> {
fn reconstruct_genesis_state(database: &mut SnapshotDatabase, path: &str) -> Result<()> {
fn cleanup_encoding(input: &'_ str) -> &'_ str {
input
.strip_prefix("E'\\\\x")
Expand Down Expand Up @@ -236,7 +237,7 @@ mod tests {
use std::fs;

use indexmap::IndexMap;
use state_reconstruct_storage::{DBMode, InnerDB, PackingType};
use state_reconstruct_storage::PackingType;

use super::*;

Expand Down Expand Up @@ -271,7 +272,7 @@ mod tests {
builder.run(rx).await;
}

let db = InnerDB::new(PathBuf::from(db_dir.clone()), DBMode::Snapshot).unwrap();
let db = SnapshotDatabase::new(PathBuf::from(db_dir.clone())).unwrap();

let key = U256::from_dec_str("1234").unwrap();
let Some(log) = db.get_storage_log(&key).unwrap() else {
Expand Down
8 changes: 4 additions & 4 deletions src/processor/tree/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ use state_reconstruct_fetcher::{
metrics::{PerfMetric, METRICS_TRACING_TARGET},
types::CommitBlock,
};
use state_reconstruct_storage::{DBMode::Reconstruction, InnerDB};
use state_reconstruct_storage::reconstruction::ReconstructionDatabase;
use tokio::{
sync::{mpsc, Mutex},
time::Instant,
Expand All @@ -26,7 +26,7 @@ pub struct TreeProcessor {
/// The internal merkle tree.
tree: TreeWrapper,
/// The stored state snapshot.
inner_db: Arc<Mutex<InnerDB>>,
inner_db: Arc<Mutex<ReconstructionDatabase>>,
}

impl TreeProcessor {
Expand All @@ -46,14 +46,14 @@ impl TreeProcessor {
);
}

let new_state = InnerDB::new(inner_db_path.clone(), Reconstruction)?;
let new_state = ReconstructionDatabase::new(inner_db_path.clone())?;
let inner_db = Arc::new(Mutex::new(new_state));
let tree = TreeWrapper::new(&db_path, inner_db.clone(), init).await?;

Ok(Self { tree, inner_db })
}

pub fn get_inner_db(&self) -> Arc<Mutex<InnerDB>> {
pub fn get_inner_db(&self) -> Arc<Mutex<ReconstructionDatabase>> {
self.inner_db.clone()
}
}
Expand Down
10 changes: 6 additions & 4 deletions src/processor/tree/tree_wrapper.rs
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,9 @@ use blake2::{Blake2s256, Digest};
use ethers::types::{Address, H256, U256, U64};
use eyre::Result;
use state_reconstruct_fetcher::{constants::storage::INITAL_STATE_PATH, types::CommitBlock};
use state_reconstruct_storage::{types::SnapshotStorageLogsChunk, InnerDB, PackingType};
use state_reconstruct_storage::{
reconstruction::ReconstructionDatabase, types::SnapshotStorageLogsChunk, PackingType,
};
use thiserror::Error;
use tokio::sync::Mutex;
use zksync_merkle_tree::{Database, MerkleTree, RocksDBWrapper, TreeEntry};
Expand All @@ -22,14 +24,14 @@ pub struct TreeWrapper {
index_to_key: HashMap<u64, U256>,
key_to_value: HashMap<U256, H256>,
tree: MerkleTree<RocksDBWrapper>,
inner_db: Arc<Mutex<InnerDB>>,
inner_db: Arc<Mutex<ReconstructionDatabase>>,
}

impl TreeWrapper {
/// Attempts to create a new [`TreeWrapper`].
pub async fn new(
db_path: &Path,
inner_db: Arc<Mutex<InnerDB>>,
inner_db: Arc<Mutex<ReconstructionDatabase>>,
reconstruct: bool,
) -> Result<Self> {
let db_opt = RocksDBOptions {
Expand Down Expand Up @@ -199,7 +201,7 @@ impl TreeWrapper {
/// Attempts to reconstruct the genesis state from a CSV file.
fn reconstruct_genesis_state<D: Database>(
tree: &mut MerkleTree<D>,
snapshot: &mut InnerDB,
snapshot: &mut ReconstructionDatabase,
path: &str,
) -> Result<()> {
fn cleanup_encoding(input: &'_ str) -> &'_ str {
Expand Down
9 changes: 6 additions & 3 deletions state-reconstruct-fetcher/src/l1_fetcher.rs
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ use ethers::{
};
use eyre::Result;
use rand::random;
use state_reconstruct_storage::InnerDB;
use state_reconstruct_storage::reconstruction::ReconstructionDatabase;
use thiserror::Error;
use tokio::{
sync::{mpsc, Mutex},
Expand Down Expand Up @@ -68,12 +68,15 @@ pub struct L1Fetcher {
provider: Provider<Http>,
contracts: Contracts,
config: L1FetcherOptions,
inner_db: Option<Arc<Mutex<InnerDB>>>,
inner_db: Option<Arc<Mutex<ReconstructionDatabase>>>,
metrics: Arc<Mutex<L1Metrics>>,
}

impl L1Fetcher {
pub fn new(config: L1FetcherOptions, inner_db: Option<Arc<Mutex<InnerDB>>>) -> Result<Self> {
pub fn new(
config: L1FetcherOptions,
inner_db: Option<Arc<Mutex<ReconstructionDatabase>>>,
) -> Result<Self> {
let provider = Provider::<Http>::try_from(&config.http_url)
.expect("could not instantiate HTTP Provider");

Expand Down
Loading

0 comments on commit ea60992

Please sign in to comment.