From ed0ef8024c75e96f4afa1b5c63166b9dedc6ca7b Mon Sep 17 00:00:00 2001 From: Liu-Cheng Xu Date: Thu, 4 Jul 2024 11:45:27 +0800 Subject: [PATCH 1/3] Set flag --execute-block as true by default --- .../src/commands/import_blocks.rs | 24 +++++++++++++------ 1 file changed, 17 insertions(+), 7 deletions(-) diff --git a/crates/subcoin-node/src/commands/import_blocks.rs b/crates/subcoin-node/src/commands/import_blocks.rs index 6d491f30..f3779dc4 100644 --- a/crates/subcoin-node/src/commands/import_blocks.rs +++ b/crates/subcoin-node/src/commands/import_blocks.rs @@ -1,6 +1,6 @@ use crate::cli::params::CommonParams; use bitcoin_explorer::BitcoinDB; -use sc_cli::{NodeKeyParams, SharedParams}; +use sc_cli::{ImportParams, NodeKeyParams, SharedParams}; use sc_client_api::HeaderBackend; use sc_consensus_nakamoto::{ BitcoinBlockImport, BitcoinBlockImporter, BlockVerification, ImportConfig, @@ -30,9 +30,13 @@ pub struct ImportBlocks { pub to: Option, /// Whether to execute the transactions in the block. - #[clap(long)] + #[clap(long, default_value_t = true)] pub execute_block: bool, + #[allow(missing_docs)] + #[clap(flatten)] + pub import_params: ImportParams, + #[allow(missing_docs)] #[clap(flatten)] pub common_params: CommonParams, @@ -41,6 +45,7 @@ pub struct ImportBlocks { /// Custom version of [`sc_cli::ImportBlocksCmd`]. pub struct ImportBlocksCmd { shared_params: SharedParams, + import_params: ImportParams, to: Option, execute_block: bool, } @@ -49,8 +54,10 @@ impl ImportBlocksCmd { /// Constructs a new instance of [`ImportBlocksCmd`]. pub fn new(cmd: &ImportBlocks) -> Self { let shared_params = cmd.common_params.as_shared_params(); + let import_params = cmd.import_params.clone(); Self { shared_params, + import_params, to: cmd.to, execute_block: cmd.execute_block, } @@ -106,18 +113,17 @@ impl ImportBlocksCmd { if total_imported > 0 { let info = client.info(); + let best_number = info.best_number; + let substrate_block_hash = info.best_hash; + let bitcoin_block_hash = BackendExt::::bitcoin_block_hash_for(&client, info.best_hash) .unwrap_or_else(|| { panic!( - "bitcoin block hash for substrate#{},{} is missing", - info.best_number, info.best_hash + "Bitcoin block hash for substrate#{best_number},{substrate_block_hash} is missing", ) }); - let best_number = info.best_number; - let substrate_block_hash = info.best_hash; - let speed = speed::(best_number, last_number, last_update); tracing::info!( @@ -194,6 +200,10 @@ impl sc_cli::CliConfiguration for ImportBlocksCmd { &self.shared_params } + fn import_params(&self) -> Option<&ImportParams> { + Some(&self.import_params) + } + fn node_key_params(&self) -> Option<&NodeKeyParams> { None } From 2b4e5f43daa4a24440dd617e64823c88290a49ce Mon Sep 17 00:00:00 2001 From: Liu-Cheng Xu Date: Thu, 4 Jul 2024 11:45:56 +0800 Subject: [PATCH 2/3] Ignore *.log --- .gitignore | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitignore b/.gitignore index 73fab072..83a8dd89 100644 --- a/.gitignore +++ b/.gitignore @@ -8,3 +8,5 @@ target/ # MSVC Windows builds of rustc generate these, which store debugging information *.pdb + +*.log From 8b805e22f97b456afac97077f8777bec8a80f9f1 Mon Sep 17 00:00:00 2001 From: Liu-Cheng Xu Date: Fri, 5 Jul 2024 16:06:36 +0800 Subject: [PATCH 3/3] Finalize blocks with enough confirmations It's observed that the memory usage could be extremely high without the finalization, when the chain grows to 220000+. Concretely, the culprit of the high memory usage is creating `NonCanonicalOverlay`. There are also a few other improvements to import-blocks command. --- .../sc-consensus-nakamoto/src/block_import.rs | 6 ++-- crates/subcoin-node/src/cli.rs | 13 ++++++++ .../src/commands/import_blocks.rs | 31 +++++++++++++------ 3 files changed, 39 insertions(+), 11 deletions(-) diff --git a/crates/sc-consensus-nakamoto/src/block_import.rs b/crates/sc-consensus-nakamoto/src/block_import.rs index d2c40396..9a82d809 100644 --- a/crates/sc-consensus-nakamoto/src/block_import.rs +++ b/crates/sc-consensus-nakamoto/src/block_import.rs @@ -267,10 +267,11 @@ where /// Result of the operation of importing a Bitcoin block. /// -/// Same semantic with [`sc_consensus::ImportResult`] for including more information -/// in the Imported variant. +/// Same semantic with [`sc_consensus::ImportResult`] with additional information +/// in the [`ImportStatus::Imported`] variant. #[derive(Debug, Clone)] pub enum ImportStatus { + /// Block was imported successfully. Imported { block_number: u32, block_hash: BlockHash, @@ -287,6 +288,7 @@ pub enum ImportStatus { } impl ImportStatus { + /// Returns `true` if the import status is [`Self::UnknownParent`]. pub fn is_unknown_parent(&self) -> bool { matches!(self, Self::UnknownParent) } diff --git a/crates/subcoin-node/src/cli.rs b/crates/subcoin-node/src/cli.rs index f75d1d6e..7174ec28 100644 --- a/crates/subcoin-node/src/cli.rs +++ b/crates/subcoin-node/src/cli.rs @@ -60,6 +60,19 @@ pub fn run() -> sc_cli::Result<()> { network: bitcoin::Network::Bitcoin, config: &config, })?; + task_manager.spawn_handle().spawn("finalizer", None, { + let client = client.clone(); + let spawn_handle = task_manager.spawn_handle(); + let confirmation_depth = 6u32; + // TODO: proper value + let is_major_syncing = std::sync::Arc::new(true.into()); + subcoin_service::finalize_confirmed_blocks( + client, + spawn_handle, + confirmation_depth, + is_major_syncing, + ) + }); Ok((import_blocks_cmd.run(client, data_dir), task_manager)) }) } diff --git a/crates/subcoin-node/src/commands/import_blocks.rs b/crates/subcoin-node/src/commands/import_blocks.rs index f3779dc4..3159df54 100644 --- a/crates/subcoin-node/src/commands/import_blocks.rs +++ b/crates/subcoin-node/src/commands/import_blocks.rs @@ -17,21 +17,26 @@ use subcoin_service::FullClient; /// Import Bitcoin blocks from bitcoind database. #[derive(clap::Parser, Debug, Clone)] pub struct ImportBlocks { - /// Path of the bitcoind database. + /// Path to the bitcoind database. /// - /// Value of the `-data-dir` argument in the bitcoind program. + /// This corresponds to the value of the `-data-dir` argument in the bitcoind program. #[clap(index = 1, value_parser)] pub data_dir: PathBuf, - /// Specify the block number of last block to import. + /// Number of blocks to import. /// - /// The default value is the highest block in the database. + /// The process will stop after importing the specified number of blocks. + pub block_count: Option, + + /// Block number of last block to import. + /// + /// The default value is to the highest block in the database. #[clap(long)] - pub to: Option, + pub end_block: Option, - /// Whether to execute the transactions in the block. + /// Whether to execute the transactions within the blocks. #[clap(long, default_value_t = true)] - pub execute_block: bool, + pub execute_transactions: bool, #[allow(missing_docs)] #[clap(flatten)] @@ -46,6 +51,7 @@ pub struct ImportBlocks { pub struct ImportBlocksCmd { shared_params: SharedParams, import_params: ImportParams, + block_count: Option, to: Option, execute_block: bool, } @@ -58,8 +64,9 @@ impl ImportBlocksCmd { Self { shared_params, import_params, - to: cmd.to, - execute_block: cmd.execute_block, + block_count: cmd.block_count, + to: cmd.end_block, + execute_block: cmd.execute_transactions, } } @@ -142,6 +149,12 @@ impl ImportBlocksCmd { } total_imported += 1; + + if let Some(block_count) = self.block_count { + if total_imported == block_count { + break; + } + } } tracing::info!("Imported {total_imported} blocks successfully");