Merge pull request #45 from alsrdn/track-feat-2.0

Refactor `casper-db-utils` to make it compatible with casper-node 2.0

Cargo.toml

@@ -14,30 +14,33 @@ build = "build.rs"
 [dependencies]
 anyhow = "1"
 bincode = "1"
-casper-execution-engine = "4"
-casper-hashing = "1.4"
-casper-node = "=1.4.15-alt"
-casper-types = "2"
+casper-execution-engine = "*"
+casper-storage = { git = "https://github.com/casper-network/casper-node.git", branch = "feat-2.0" }
+casper-types = "*"
 clap = { version = "3", features = ["cargo"] }
 futures = "0.3.21"
-lmdb = "0.8.0"
-lmdb-sys = "0.8.0"
+lmdb-rkv = "0.14"
 log = "0.4.17"
 once_cell = "1"
-reqwest = { version = "0.11.10", features = ["stream"] }
+reqwest = { version = "0.12.4", features = ["stream"] }
 ringbuf = "0.2.8"
 serde = { version = "1", features = ["derive"] }
 serde_json = "1"
 simplelog = "0.12.0"
 tar = "0.4.38"
 thiserror = "1"
 tokio = { version = "1", features = ["full"] }
-zstd = "0.12"
+zstd = "0.13.1"
 
 [dev-dependencies]
 once_cell = "1"
 rand = "0.8.5"
 tempfile = "3"
 
 [build-dependencies]
-cargo-lock = { version = "9.0", default-features = false }
+cargo-lock = { version = "9.0", default-features = false }
+
+[patch.crates-io]
+casper-types = { git = "https://github.com/casper-network/casper-node.git", branch = "feat-2.0" }
+casper-execution-engine = { git = "https://github.com/casper-network/casper-node.git", branch = "feat-2.0" }
+

src/common.rs

@@ -1,3 +1,2 @@
 pub mod db;
-pub mod lmdb_utils;
 pub mod progress;

src/common/db.rs

@@ -1,30 +1,41 @@
+mod approvals_hashes_db;
 mod block_body_db;
-mod block_body_merkle_db;
+mod block_body_v2_db;
 mod block_header_db;
+mod block_header_v2;
 mod block_metadata_db;
-mod deploy_hashes_db;
+mod block_metadata_v2_db;
 mod deploy_metadata_db;
 mod deploys_db;
+mod execution_results_db;
 mod finalized_approvals_db;
-mod proposers_db;
 mod state_store_db;
+mod transactions_db;
+mod transfer_db;
+mod versioned_approvals_hashes_db;
+mod versioned_finalized_approvals_db;
+mod versioned_transfers_db;
+
 #[cfg(test)]
 mod tests;
-mod transfer_db;
-mod transfer_hashes_db;
-
-pub use block_body_db::BlockBodyDatabase;
-pub use block_body_merkle_db::BlockBodyMerkleDatabase;
-pub use block_header_db::BlockHeaderDatabase;
-pub use block_metadata_db::BlockMetadataDatabase;
-pub use deploy_hashes_db::DeployHashesDatabase;
-pub use deploy_metadata_db::DeployMetadataDatabase;
+
+pub use approvals_hashes_db::ApprovalsHashesDatabase;
+pub use block_body_db::LegacyBlockBodyDatabase;
+pub use block_body_v2_db::VersionedBlockBodyDatabase;
+pub use block_header_db::LegacyBlockHeaderDatabase;
+pub use block_header_v2::VersionedBlockHeaderDatabase;
+pub use block_metadata_db::LegacyBlockMetadataDatabase;
+pub use block_metadata_v2_db::VersionedBlockMetadataDatabase;
+pub use deploy_metadata_db::LegacyDeployMetadataDatabase;
 pub use deploys_db::DeployDatabase;
+pub use execution_results_db::VersionedExecutionResultsDatabase;
 pub use finalized_approvals_db::FinalizedApprovalsDatabase;
-pub use proposers_db::ProposerDatabase;
 pub use state_store_db::StateStoreDatabase;
+pub use transactions_db::TransactionsDatabase;
 pub use transfer_db::TransferDatabase;
-pub use transfer_hashes_db::TransferHashesDatabase;
+pub use versioned_approvals_hashes_db::VersionedApprovalsHashesDatabase;
+pub use versioned_finalized_approvals_db::VersionedFinalizedApprovalsDatabase;
+pub use versioned_transfers_db::VersionedTransfersDatabase;
 
 use std::{
     fmt::{Display, Formatter, Result as FormatterResult},
@@ -44,6 +55,11 @@ pub const TRIE_STORE_FILE_NAME: &str = "data.lmdb";
 const ENTRY_LOG_INTERVAL: usize = 100_000;
 const MAX_DB_READERS: u32 = 100;
 
+const GIB: usize = 1024 * 1024 * 1024;
+pub(crate) const DEFAULT_MAX_BLOCK_STORE_SIZE: usize = 450 * GIB;
+pub(crate) const DEFAULT_MAX_DEPLOY_STORE_SIZE: usize = 300 * GIB;
+pub(crate) const DEFAULT_MAX_DEPLOY_METADATA_STORE_SIZE: usize = 300 * GIB;
+
 #[derive(Debug, Error)]
 pub enum DeserializationError {
     #[error("failed parsing struct with bincode")]
@@ -109,7 +125,8 @@ pub trait Database {
             info!("Skipping {} entries.", start_at);
         }
         let mut error_buffer = vec![];
-        for (idx, (_raw_key, raw_val)) in cursor.iter().skip(start_at).enumerate() {
+        for (idx, entry) in cursor.iter().skip(start_at).enumerate() {
+            let (_raw_key, raw_val) = entry.map_err(Error::Database)?;
             if let Err(e) =
                 Self::parse_element(raw_val).map_err(|parsing_err| Error::Parsing(idx, parsing_err))
             {

src/common/db/approvals_hashes_db.rs

@@ -0,0 +1,35 @@
+use std::{
+    fmt::{Display, Formatter, Result as FormatterResult},
+    result::Result,
+};
+
+#[derive(Deserialize)]
+struct LegacyApprovalsHashes {
+    _block_hash: BlockHash,
+    _approvals_hashes: Vec<ApprovalsHash>,
+    _merkle_proof_approvals: TrieMerkleProof<Key, StoredValue>,
+}
+
+use casper_types::{global_state::TrieMerkleProof, ApprovalsHash, BlockHash, Key, StoredValue};
+use serde::Deserialize;
+
+use super::{Database, DeserializationError};
+
+pub struct ApprovalsHashesDatabase;
+
+impl Display for ApprovalsHashesDatabase {
+    fn fmt(&self, f: &mut Formatter<'_>) -> FormatterResult {
+        write!(f, "approvals_hashes")
+    }
+}
+
+impl Database for ApprovalsHashesDatabase {
+    fn db_name() -> &'static str {
+        "approvals_hashes"
+    }
+
+    fn parse_element(bytes: &[u8]) -> Result<(), DeserializationError> {
+        let _: LegacyApprovalsHashes = bincode::deserialize(bytes)?;
+        Ok(())
+    }
+}

src/common/db/block_body_db.rs

@@ -3,25 +3,25 @@ use std::{
     result::Result,
 };
 
-use casper_node::types::BlockBody;
+use casper_types::BlockBodyV1;
 
 use super::{Database, DeserializationError};
 
-pub struct BlockBodyDatabase;
+pub struct LegacyBlockBodyDatabase;
 
-impl Display for BlockBodyDatabase {
+impl Display for LegacyBlockBodyDatabase {
     fn fmt(&self, f: &mut Formatter<'_>) -> FormatterResult {
         write!(f, "block_body")
     }
 }
 
-impl Database for BlockBodyDatabase {
+impl Database for LegacyBlockBodyDatabase {
     fn db_name() -> &'static str {
         "block_body"
     }
 
     fn parse_element(bytes: &[u8]) -> Result<(), DeserializationError> {
-        let _: BlockBody = bincode::deserialize(bytes)?;
+        let _: BlockBodyV1 = bincode::deserialize(bytes)?;
         Ok(())
     }
 }

src/common/db/deploy_hashes_db.rs → src/common/db/block_body_v2_db.rs

@@ -3,25 +3,25 @@ use std::{
     result::Result,
 };
 
-use casper_types::{bytesrepr::FromBytes, DeployHash};
+use casper_types::{bytesrepr::FromBytes, BlockBody};
 
 use super::{Database, DeserializationError};
 
-pub struct DeployHashesDatabase;
+pub struct VersionedBlockBodyDatabase;
 
-impl Display for DeployHashesDatabase {
+impl Display for VersionedBlockBodyDatabase {
     fn fmt(&self, f: &mut Formatter<'_>) -> FormatterResult {
-        write!(f, "deploy_hashes")
+        write!(f, "block_body_v2")
     }
 }
 
-impl Database for DeployHashesDatabase {
+impl Database for VersionedBlockBodyDatabase {
     fn db_name() -> &'static str {
-        "deploy_hashes"
+        "block_body_v2"
     }
 
     fn parse_element(bytes: &[u8]) -> Result<(), DeserializationError> {
-        let _: Vec<DeployHash> = FromBytes::from_bytes(bytes)?.0;
+        let _: BlockBody = FromBytes::from_bytes(bytes)?.0;
         Ok(())
     }
 }

src/common/db/block_header_db.rs

@@ -3,25 +3,25 @@ use std::{
     result::Result,
 };
 
-use casper_node::types::BlockHeader;
+use casper_types::BlockHeaderV1;
 
 use super::{Database, DeserializationError};
 
-pub struct BlockHeaderDatabase;
+pub struct LegacyBlockHeaderDatabase;
 
-impl Display for BlockHeaderDatabase {
+impl Display for LegacyBlockHeaderDatabase {
     fn fmt(&self, f: &mut Formatter<'_>) -> FormatterResult {
         write!(f, "block_header")
     }
 }
 
-impl Database for BlockHeaderDatabase {
+impl Database for LegacyBlockHeaderDatabase {
     fn db_name() -> &'static str {
         "block_header"
     }
 
     fn parse_element(bytes: &[u8]) -> Result<(), DeserializationError> {
-        let _: BlockHeader = bincode::deserialize(bytes)?;
+        let _: BlockHeaderV1 = bincode::deserialize(bytes)?;
         Ok(())
     }
 }

src/common/db/transfer_hashes_db.rs → src/common/db/block_header_v2.rs

@@ -3,25 +3,25 @@ use std::{
     result::Result,
 };
 
-use casper_types::{bytesrepr::FromBytes, DeployHash};
+use casper_types::{bytesrepr::FromBytes, BlockHeader};
 
 use super::{Database, DeserializationError};
 
-pub struct TransferHashesDatabase;
+pub struct VersionedBlockHeaderDatabase;
 
-impl Display for TransferHashesDatabase {
+impl Display for VersionedBlockHeaderDatabase {
     fn fmt(&self, f: &mut Formatter<'_>) -> FormatterResult {
-        write!(f, "transfer_hashes")
+        write!(f, "block_header_v2")
     }
 }
 
-impl Database for TransferHashesDatabase {
+impl Database for VersionedBlockHeaderDatabase {
     fn db_name() -> &'static str {
-        "transfer_hashes"
+        "block_header_v2"
     }
 
     fn parse_element(bytes: &[u8]) -> Result<(), DeserializationError> {
-        let _: Vec<DeployHash> = FromBytes::from_bytes(bytes)?.0;
+        let _: BlockHeader = FromBytes::from_bytes(bytes)?.0;
         Ok(())
     }
 }

src/common/db/block_metadata_db.rs

@@ -3,25 +3,25 @@ use std::{
     result::Result,
 };
 
-use casper_node::types::BlockSignatures;
+use casper_types::BlockSignaturesV1;
 
 use super::{Database, DeserializationError};
 
-pub struct BlockMetadataDatabase;
+pub struct LegacyBlockMetadataDatabase;
 
-impl Display for BlockMetadataDatabase {
+impl Display for LegacyBlockMetadataDatabase {
     fn fmt(&self, f: &mut Formatter<'_>) -> FormatterResult {
         write!(f, "block_metadata")
     }
 }
 
-impl Database for BlockMetadataDatabase {
+impl Database for LegacyBlockMetadataDatabase {
     fn db_name() -> &'static str {
         "block_metadata"
     }
 
     fn parse_element(bytes: &[u8]) -> Result<(), DeserializationError> {
-        let _: BlockSignatures = bincode::deserialize(bytes)?;
+        let _: BlockSignaturesV1 = bincode::deserialize(bytes)?;
         Ok(())
     }
 }

src/common/db/block_metadata_v2_db.rs

@@ -0,0 +1,27 @@
+use std::{
+    fmt::{Display, Formatter, Result as FormatterResult},
+    result::Result,
+};
+
+use casper_types::{bytesrepr::FromBytes, BlockSignatures};
+
+use super::{Database, DeserializationError};
+
+pub struct VersionedBlockMetadataDatabase;
+
+impl Display for VersionedBlockMetadataDatabase {
+    fn fmt(&self, f: &mut Formatter<'_>) -> FormatterResult {
+        write!(f, "block_metadata_v2")
+    }
+}
+
+impl Database for VersionedBlockMetadataDatabase {
+    fn db_name() -> &'static str {
+        "block_metadata_v2"
+    }
+
+    fn parse_element(bytes: &[u8]) -> Result<(), DeserializationError> {
+        let _: BlockSignatures = FromBytes::from_bytes(bytes)?.0;
+        Ok(())
+    }
+}

src/common/db/deploy_metadata_db.rs

@@ -1,26 +1,33 @@
-use casper_node::types::DeployMetadata;
+use casper_types::{execution::ExecutionResultV1, BlockHash};
+use serde::{Deserialize, Serialize};
 use std::{
+    collections::HashMap,
     fmt::{Display, Formatter, Result as FormatterResult},
     result::Result,
 };
 
 use super::{Database, DeserializationError};
 
-pub struct DeployMetadataDatabase;
+#[derive(Clone, Default, Serialize, Deserialize, Debug, PartialEq, Eq)]
+struct DeployMetadataV1 {
+    execution_results: HashMap<BlockHash, ExecutionResultV1>,
+}
+
+pub struct LegacyDeployMetadataDatabase;
 
-impl Display for DeployMetadataDatabase {
+impl Display for LegacyDeployMetadataDatabase {
     fn fmt(&self, f: &mut Formatter<'_>) -> FormatterResult {
         write!(f, "deploy_metadata")
     }
 }
 
-impl Database for DeployMetadataDatabase {
+impl Database for LegacyDeployMetadataDatabase {
     fn db_name() -> &'static str {
         "deploy_metadata"
     }
 
     fn parse_element(bytes: &[u8]) -> Result<(), DeserializationError> {
-        let _: DeployMetadata = bincode::deserialize(bytes)?;
+        let _: DeployMetadataV1 = bincode::deserialize(bytes)?;
         Ok(())
     }
 }

src/common/db/deploys_db.rs

@@ -3,7 +3,7 @@ use std::{
     result::Result,
 };
 
-use casper_node::types::Deploy;
+use casper_types::Deploy;
 
 use super::{Database, DeserializationError};