From 1e59a2d7aa8ad1e75c9f11eb61a1d78d64bd3f16 Mon Sep 17 00:00:00 2001 From: Liu-Cheng Xu Date: Tue, 12 Nov 2024 09:38:54 +0800 Subject: [PATCH] Introduce subcoin-utxo-snapshot --- Cargo.lock | 12 ++ Cargo.toml | 2 + crates/subcoin-crypto/src/muhash.rs | 11 ++ crates/subcoin-node/Cargo.toml | 1 + .../subcoin-node/src/commands/blockchain.rs | 85 ++++-------- crates/subcoin-runtime-primitives/Cargo.toml | 2 + crates/subcoin-runtime-primitives/src/lib.rs | 3 +- crates/subcoin-utxo-snapshot/Cargo.toml | 13 ++ crates/subcoin-utxo-snapshot/src/lib.rs | 125 ++++++++++++++++++ 9 files changed, 193 insertions(+), 61 deletions(-) create mode 100644 crates/subcoin-utxo-snapshot/Cargo.toml create mode 100644 crates/subcoin-utxo-snapshot/src/lib.rs diff --git a/Cargo.lock b/Cargo.lock index b9ddb026..d685abed 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -10467,6 +10467,7 @@ dependencies = [ "subcoin-runtime", "subcoin-service", "subcoin-test-service", + "subcoin-utxo-snapshot", "substrate-build-script-utils", "substrate-frame-rpc-system", "substrate-prometheus-endpoint", @@ -10537,6 +10538,7 @@ version = "0.1.0" dependencies = [ "parity-scale-codec", "scale-info", + "serde", "sp-api", "sp-runtime", "sp-std 14.0.0 (git+https://github.com/subcoin-project/polkadot-sdk?branch=subcoin-v3)", @@ -10608,6 +10610,16 @@ dependencies = [ "tokio", ] +[[package]] +name = "subcoin-utxo-snapshot" +version = "0.1.0" +dependencies = [ + "bitcoin 0.32.2", + "serde", + "subcoin-primitives", + "txoutset", +] + [[package]] name = "substrate-bip39" version = "0.4.7" diff --git a/Cargo.toml b/Cargo.toml index 480e6527..205a9bc0 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -22,6 +22,7 @@ members = [ "crates/subcoin-runtime-primitives", "crates/subcoin-service", "crates/subcoin-test-service", + "crates/subcoin-utxo-snapshot", ] default-members = ["crates/subcoin-node"] @@ -119,6 +120,7 @@ subcoin-runtime = { path = "crates/subcoin-runtime" } subcoin-runtime-primitives = { path = "crates/subcoin-runtime-primitives", default-features = false } subcoin-service = { path = "crates/subcoin-service" } subcoin-test-service = { path = "crates/subcoin-test-service" } +subcoin-utxo-snapshot = { path = "crates/subcoin-utxo-snapshot" } [profile.release] panic = "abort" diff --git a/crates/subcoin-crypto/src/muhash.rs b/crates/subcoin-crypto/src/muhash.rs index 31d0c08c..dd0be471 100644 --- a/crates/subcoin-crypto/src/muhash.rs +++ b/crates/subcoin-crypto/src/muhash.rs @@ -4,6 +4,7 @@ use crate::chacha20_block; use num_bigint::{BigUint, ToBigUint}; use num_traits::One; use sha2::{Digest, Sha256}; +use std::fmt::Write; // Function to hash a 32-byte array into a 3072-bit number using 6 ChaCha20 operations fn data_to_num3072(data: &[u8; 32]) -> BigUint { @@ -67,6 +68,16 @@ impl MuHash3072 { bytes384.resize(384, 0); // Ensure it is exactly 384 bytes Sha256::digest(&bytes384).to_vec() } + + /// Returns the value of `muhash` in Bitcoin Core's dumptxoutset output. + pub fn txoutset_muhash(&self) -> String { + let finalized = self.digest(); + + finalized.iter().rev().fold(String::new(), |mut output, b| { + let _ = write!(output, "{b:02x}"); + output + }) + } } #[cfg(test)] diff --git a/crates/subcoin-node/Cargo.toml b/crates/subcoin-node/Cargo.toml index f7e9ea8e..036f7cde 100644 --- a/crates/subcoin-node/Cargo.toml +++ b/crates/subcoin-node/Cargo.toml @@ -52,6 +52,7 @@ subcoin-primitives = { workspace = true } subcoin-rpc = { workspace = true } subcoin-runtime = { workspace = true } subcoin-service = { workspace = true } +subcoin-utxo-snapshot = { workspace = true } substrate-frame-rpc-system = { workspace = true } substrate-prometheus-endpoint = { workspace = true } tracing = { workspace = true } diff --git a/crates/subcoin-node/src/commands/blockchain.rs b/crates/subcoin-node/src/commands/blockchain.rs index 413c5f7c..d8043985 100644 --- a/crates/subcoin-node/src/commands/blockchain.rs +++ b/crates/subcoin-node/src/commands/blockchain.rs @@ -6,7 +6,6 @@ use sc_client_api::{HeaderBackend, StorageProvider}; use serde::Serialize; use sp_core::storage::StorageKey; use sp_core::Decode; -use std::fmt::Write as _; use std::fs::File; use std::io::{Stdout, Write}; use std::path::PathBuf; @@ -15,6 +14,7 @@ use std::time::{Duration, Instant}; use subcoin_primitives::runtime::Coin; use subcoin_primitives::{BackendExt, CoinStorageKey}; use subcoin_service::FullClient; +use subcoin_utxo_snapshot::UtxoSnapshotGenerator; const FINAL_STORAGE_PREFIX_LEN: usize = 32; @@ -284,30 +284,6 @@ fn fetch_utxo_set_at( )) } -// Equivalent function in Rust for serializing an OutPoint and Coin -// -// https://github.com/bitcoin/bitcoin/blob/6f9db1ebcab4064065ccd787161bf2b87e03cc1f/src/kernel/coinstats.cpp#L51 -fn tx_out_ser(outpoint: bitcoin::OutPoint, coin: &Coin) -> bitcoin::io::Result> { - let mut data = Vec::new(); - - // Serialize the OutPoint (txid and vout) - outpoint.consensus_encode(&mut data)?; - - // Serialize the coin's height and coinbase flag - let height_and_coinbase = (coin.height << 1) | (coin.is_coinbase as u32); - height_and_coinbase.consensus_encode(&mut data)?; - - let txout = bitcoin::TxOut { - value: bitcoin::Amount::from_sat(coin.amount), - script_pubkey: bitcoin::ScriptBuf::from_bytes(coin.script_pubkey.clone()), - }; - - // Serialize the actual UTXO (value and script) - txout.consensus_encode(&mut data)?; - - Ok(data) -} - // Custom serializer for total_amount to display 8 decimal places fn serialize_as_btc(amount: &u64, serializer: S) -> Result where @@ -350,7 +326,7 @@ async fn gettxoutsetinfo( let mut muhash = subcoin_crypto::muhash::MuHash3072::new(); for (txid, vout, coin) in utxo_iter { - let data = tx_out_ser(bitcoin::OutPoint { txid, vout }, &coin) + let data = subcoin_utxo_snapshot::tx_out_ser(bitcoin::OutPoint { txid, vout }, &coin) .map_err(|err| sc_cli::Error::Application(Box::new(err)))?; muhash.insert(&data); @@ -375,19 +351,14 @@ async fn gettxoutsetinfo( } // Hash the combined hash of all UTXOs - let finalized = muhash.digest(); - - let utxo_set_hash = finalized.iter().rev().fold(String::new(), |mut output, b| { - let _ = write!(output, "{b:02x}"); - output - }); + let muhash = muhash.txoutset_muhash(); let tx_out_set_info = TxOutSetInfo { height: block_number, bestblock: bitcoin_block_hash, txouts, bogosize, - muhash: utxo_set_hash, + muhash, total_amount, }; @@ -492,7 +463,7 @@ async fn dumptxoutset( let _ = file.write(data.as_slice())?; - UtxoSetOutput::Binary(file) + UtxoSetOutput::Snapshot(UtxoSnapshotGenerator::new(file)) } else { println!("Dumping UTXO set at #{block_number},{bitcoin_block_hash}"); UtxoSetOutput::Stdout(std::io::stdout()) @@ -509,42 +480,27 @@ async fn dumptxoutset( } enum UtxoSetOutput { - Binary(File), + Snapshot(UtxoSnapshotGenerator), Csv(File), Stdout(Stdout), } impl UtxoSetOutput { fn write(&mut self, txid: bitcoin::Txid, vout: u32, coin: Coin) -> std::io::Result<()> { - let Coin { - is_coinbase, - amount, - height, - script_pubkey, - } = coin; - - let outpoint = bitcoin::OutPoint { txid, vout }; - match self { - Self::Binary(ref mut file) => { - let mut data = Vec::new(); - - let amount = txoutset::Amount::new(amount); - - let code = txoutset::Code { - height, + Self::Snapshot(snapshot_generator) => { + snapshot_generator.write_utxo_entry(txid, vout, coin)?; + } + Self::Csv(ref mut file) => { + let Coin { is_coinbase, - }; - let script = txoutset::Script::from_bytes(script_pubkey); + amount, + height, + script_pubkey, + } = coin; - outpoint.consensus_encode(&mut data)?; - code.consensus_encode(&mut data)?; - amount.consensus_encode(&mut data)?; - script.consensus_encode(&mut data)?; + let outpoint = bitcoin::OutPoint { txid, vout }; - let _ = file.write(data.as_slice())?; - } - Self::Csv(ref mut file) => { let script_pubkey = hex::encode(script_pubkey.as_slice()); writeln!( file, @@ -552,6 +508,15 @@ impl UtxoSetOutput { )?; } Self::Stdout(ref mut stdout) => { + let Coin { + is_coinbase, + amount, + height, + script_pubkey, + } = coin; + + let outpoint = bitcoin::OutPoint { txid, vout }; + let script_pubkey = hex::encode(script_pubkey.as_slice()); writeln!( stdout, diff --git a/crates/subcoin-runtime-primitives/Cargo.toml b/crates/subcoin-runtime-primitives/Cargo.toml index 02d21b42..3449033a 100644 --- a/crates/subcoin-runtime-primitives/Cargo.toml +++ b/crates/subcoin-runtime-primitives/Cargo.toml @@ -13,12 +13,14 @@ scale-info = { workspace = true, default-features = false } sp-api = { workspace = true, default-features = false } sp-runtime = { workspace = true, default-features = false } sp-std = { workspace = true, default-features = false } +serde = { workspace = true, optional = true } [features] default = ["std"] std = [ "codec/std", "scale-info/std", + "serde", "sp-api/std", "sp-runtime/std", "sp-std/std", diff --git a/crates/subcoin-runtime-primitives/src/lib.rs b/crates/subcoin-runtime-primitives/src/lib.rs index 1c6de6fe..4c03acd6 100644 --- a/crates/subcoin-runtime-primitives/src/lib.rs +++ b/crates/subcoin-runtime-primitives/src/lib.rs @@ -26,7 +26,8 @@ const HALVING_INTERVAL: u32 = 210_000; const MAX_SCRIPT_SIZE: usize = 10_000; /// Unspent transaction output. -#[derive(Debug, TypeInfo, Encode, Decode)] +#[derive(Debug, Clone, PartialEq, Eq, TypeInfo, Encode, Decode)] +#[cfg_attr(feature = "std", derive(serde::Serialize, serde::Deserialize))] pub struct Coin { /// Whether the coin is from a coinbase transaction. pub is_coinbase: bool, diff --git a/crates/subcoin-utxo-snapshot/Cargo.toml b/crates/subcoin-utxo-snapshot/Cargo.toml new file mode 100644 index 00000000..d183bca2 --- /dev/null +++ b/crates/subcoin-utxo-snapshot/Cargo.toml @@ -0,0 +1,13 @@ +[package] +name = "subcoin-utxo-snapshot" +version.workspace = true +authors.workspace = true +edition.workspace = true +repository.workspace = true +license.workspace = true + +[dependencies] +bitcoin = { workspace = true } +serde = { workspace = true } +subcoin-primitives = { workspace = true } +txoutset = { workspace = true } diff --git a/crates/subcoin-utxo-snapshot/src/lib.rs b/crates/subcoin-utxo-snapshot/src/lib.rs new file mode 100644 index 00000000..f43c3c8a --- /dev/null +++ b/crates/subcoin-utxo-snapshot/src/lib.rs @@ -0,0 +1,125 @@ +use bitcoin::consensus::encode::Encodable; +use bitcoin::BlockHash; +use std::fs::File; +use std::io::Write; +use subcoin_primitives::runtime::Coin; + +// Equivalent function in Rust for serializing an OutPoint and Coin +// +// https://github.com/bitcoin/bitcoin/blob/6f9db1ebcab4064065ccd787161bf2b87e03cc1f/src/kernel/coinstats.cpp#L51 +pub fn tx_out_ser(outpoint: bitcoin::OutPoint, coin: &Coin) -> bitcoin::io::Result> { + let mut data = Vec::new(); + + // Serialize the OutPoint (txid and vout) + outpoint.consensus_encode(&mut data)?; + + // Serialize the coin's height and coinbase flag + let height_and_coinbase = (coin.height << 1) | (coin.is_coinbase as u32); + height_and_coinbase.consensus_encode(&mut data)?; + + let txout = bitcoin::TxOut { + value: bitcoin::Amount::from_sat(coin.amount), + script_pubkey: bitcoin::ScriptBuf::from_bytes(coin.script_pubkey.clone()), + }; + + // Serialize the actual UTXO (value and script) + txout.consensus_encode(&mut data)?; + + Ok(data) +} + +/// Represents a single UTXO (Unspent Transaction Output) in Bitcoin. +#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)] +pub struct Utxo { + /// The transaction ID that contains this UTXO. + pub txid: bitcoin::Txid, + /// The output index within the transaction. + pub vout: u32, + /// The coin data associated with this UTXO (e.g., amount and any relevant metadata). + pub coin: Coin, +} + +/// Responsible for dumping the UTXO set snapshot compatible with Bitcoin Core. +pub struct UtxoSnapshotGenerator { + output_file: File, +} + +impl UtxoSnapshotGenerator { + /// Constructs a new instance of [`UtxoSnapshotGenerator`]. + pub fn new(output_file: File) -> Self { + Self { output_file } + } + + /// Writes a single entry of UTXO. + pub fn write_utxo_entry( + &mut self, + txid: bitcoin::Txid, + vout: u32, + coin: Coin, + ) -> std::io::Result<()> { + let Coin { + is_coinbase, + amount, + height, + script_pubkey, + } = coin; + + let outpoint = bitcoin::OutPoint { txid, vout }; + + let mut data = Vec::new(); + + let amount = txoutset::Amount::new(amount); + + let code = txoutset::Code { + height, + is_coinbase, + }; + let script = txoutset::Script::from_bytes(script_pubkey); + + outpoint.consensus_encode(&mut data)?; + code.consensus_encode(&mut data)?; + amount.consensus_encode(&mut data)?; + script.consensus_encode(&mut data)?; + + let _ = self.output_file.write(data.as_slice())?; + + Ok(()) + } + + /// Writes the metadata of snapshot. + pub fn write_snapshot_metadata( + &mut self, + bitcoin_block_hash: BlockHash, + coins_count: u64, + ) -> std::io::Result<()> { + let mut data = Vec::new(); + + bitcoin_block_hash + .consensus_encode(&mut data) + .expect("Failed to encode"); + + coins_count + .consensus_encode(&mut data) + .expect("Failed to write utxo set size"); + + let _ = self.output_file.write(data.as_slice())?; + + Ok(()) + } + + /// Write the UTXO snapshot at the specified block to a file. + pub fn write_utxo_snapshot( + &mut self, + bitcoin_block_hash: BlockHash, + utxos_count: u64, + utxos: impl IntoIterator, + ) -> std::io::Result<()> { + self.write_snapshot_metadata(bitcoin_block_hash, utxos_count)?; + + for Utxo { txid, vout, coin } in utxos { + self.write_utxo_entry(txid, vout, coin)?; + } + + Ok(()) + } +}