From 9b3efbd648e1044e59c27dbeb2eb93ab9784d320 Mon Sep 17 00:00:00 2001 From: Matthew Dean Date: Mon, 18 Dec 2023 13:47:09 +0000 Subject: [PATCH 1/2] Add option to enable manual sealing of blocks when in dev mode --- Cargo.lock | 79 ++++++++++++- node/Cargo.toml | 4 +- node/src/cli.rs | 4 + node/src/command.rs | 10 +- node/src/main.rs | 6 +- node/src/rpc.rs | 45 +++++++- node/src/test_service.rs | 240 +++++++++++++++++++++++++++++++++++++++ runtime/Cargo.toml | 2 +- runtime/src/lib.rs | 2 +- 9 files changed, 378 insertions(+), 14 deletions(-) create mode 100644 node/src/test_service.rs diff --git a/Cargo.lock b/Cargo.lock index ca28aefa..cb082c74 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -372,6 +372,12 @@ version = "0.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "155a5a185e42c6b77ac7b88a15143d930a9e9727a5b7b77eed417404ab15c247" +[[package]] +name = "assert_matches" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b34d609dfbaf33d6889b2b7106d3ca345eacad44200913df5ba02bfd31d2ba9" + [[package]] name = "async-channel" version = "1.8.0" @@ -425,9 +431,9 @@ dependencies = [ [[package]] name = "async-trait" -version = "0.1.68" +version = "0.1.74" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b9ccdd8f2a161be9bd5c023df56f1b2a0bd1d83872ae53b71a84a12c9bf6e842" +checksum = "a66537f1bb974b254c98ed142ff995236e81b9d0fe4db0575f46612cb15eb0f9" dependencies = [ "proc-macro2", "quote", @@ -1718,8 +1724,9 @@ dependencies = [ [[package]] name = "dscp-node" -version = "9.2.0" +version = "9.3.0" dependencies = [ + "async-trait", "bs58", "clap", "dscp-lang", @@ -1738,6 +1745,7 @@ dependencies = [ "sc-consensus-babe", "sc-consensus-babe-rpc", "sc-consensus-grandpa", + "sc-consensus-manual-seal", "sc-executor", "sc-keystore", "sc-rpc", @@ -1765,7 +1773,7 @@ dependencies = [ [[package]] name = "dscp-node-runtime" -version = "9.1.2" +version = "9.3.0" dependencies = [ "dscp-runtime-types", "frame-benchmarking", @@ -6497,6 +6505,35 @@ dependencies = [ "thiserror", ] +[[package]] +name = "sc-consensus-aura" +version = "0.10.0-dev" +source = "git+https://github.com/paritytech/substrate.git?branch=polkadot-v0.9.42#ff24c60ac7d9f87727ecdd0ded9a80c56e4f4b65" +dependencies = [ + "async-trait", + "futures", + "log", + "parity-scale-codec", + "sc-block-builder", + "sc-client-api", + "sc-consensus", + "sc-consensus-slots", + "sc-telemetry", + "sp-api", + "sp-application-crypto", + "sp-block-builder", + "sp-blockchain", + "sp-consensus", + "sp-consensus-aura", + "sp-consensus-slots", + "sp-core", + "sp-inherents", + "sp-keystore", + "sp-runtime", + "substrate-prometheus-endpoint", + "thiserror", +] + [[package]] name = "sc-consensus-babe" version = "0.10.0-dev" @@ -6608,6 +6645,40 @@ dependencies = [ "thiserror", ] +[[package]] +name = "sc-consensus-manual-seal" +version = "0.10.0-dev" +source = "git+https://github.com/paritytech/substrate.git?branch=polkadot-v0.9.42#ff24c60ac7d9f87727ecdd0ded9a80c56e4f4b65" +dependencies = [ + "assert_matches", + "async-trait", + "futures", + "jsonrpsee", + "log", + "parity-scale-codec", + "sc-client-api", + "sc-consensus", + "sc-consensus-aura", + "sc-consensus-babe", + "sc-consensus-epochs", + "sc-transaction-pool", + "sc-transaction-pool-api", + "serde", + "sp-api", + "sp-blockchain", + "sp-consensus", + "sp-consensus-aura", + "sp-consensus-babe", + "sp-consensus-slots", + "sp-core", + "sp-inherents", + "sp-keystore", + "sp-runtime", + "sp-timestamp", + "substrate-prometheus-endpoint", + "thiserror", +] + [[package]] name = "sc-consensus-slots" version = "0.10.0-dev" diff --git a/node/Cargo.toml b/node/Cargo.toml index 5520e262..7cc6b3d9 100644 --- a/node/Cargo.toml +++ b/node/Cargo.toml @@ -6,7 +6,7 @@ edition = '2021' license = 'Apache-2.0' repository = 'https://github.com/digicatapult/dscp-node/' name = 'dscp-node' -version = '9.2.0' +version = '9.3.0' [[bin]] name = 'dscp-node' @@ -18,6 +18,7 @@ targets = ['x86_64-unknown-linux-gnu'] clap = { version = "4.4.6", features = ["derive"] } futures = { version = "0.3.27", features = ["thread-pool"] } bs58 = "0.4.0" +async-trait = "0.1.57" sc-cli = { version = "0.10.0-dev", git = "https://github.com/paritytech/substrate.git", branch = "polkadot-v0.9.42" } sp-core = { version = "7.0.0", git = "https://github.com/paritytech/substrate.git", branch = "polkadot-v0.9.42" } @@ -27,6 +28,7 @@ sc-telemetry = { version = "4.0.0-dev", git = "https://github.com/paritytech/sub sc-keystore = { version = "4.0.0-dev", git = "https://github.com/paritytech/substrate.git", branch = "polkadot-v0.9.42" } sc-transaction-pool = { version = "4.0.0-dev", git = "https://github.com/paritytech/substrate.git", branch = "polkadot-v0.9.42" } sc-transaction-pool-api = { version = "4.0.0-dev", git = "https://github.com/paritytech/substrate.git", branch = "polkadot-v0.9.42" } +sc-consensus-manual-seal = { version = "0.10.0-dev", git = "https://github.com/paritytech/substrate.git", branch = "polkadot-v0.9.42" } sc-consensus-babe = { version = "0.10.0-dev", git = "https://github.com/paritytech/substrate.git", branch = "polkadot-v0.9.42" } sc-consensus-babe-rpc = { version = "0.10.0-dev", git = "https://github.com/paritytech/substrate.git", branch = "polkadot-v0.9.42" } sp-consensus-babe = { version = "0.10.0-dev", git = "https://github.com/paritytech/substrate.git", branch = "polkadot-v0.9.42" } diff --git a/node/src/cli.rs b/node/src/cli.rs index 599f8db5..1572cfab 100644 --- a/node/src/cli.rs +++ b/node/src/cli.rs @@ -8,6 +8,10 @@ pub struct Cli { #[clap(flatten)] pub run: RunCmd, + + /// Enable manual sealing of blocks, primarily for testing. Must be run with --dev + #[arg(long, requires = "dev")] + pub manual_seal: bool, } #[derive(Debug, clap::Subcommand)] diff --git a/node/src/command.rs b/node/src/command.rs index e8c20b84..909e7c2f 100644 --- a/node/src/command.rs +++ b/node/src/command.rs @@ -2,7 +2,7 @@ use crate::{ benchmarking::{inherent_benchmark_data, RemarkBuilder, TransferKeepAliveBuilder}, chain_spec, cli::{Cli, Subcommand}, - service, + service, test_service, }; use dscp_node_runtime::{Block, EXISTENTIAL_DEPOSIT}; use frame_benchmarking_cli::{BenchmarkCmd, ExtrinsicFactory, SUBSTRATE_REFERENCE_HARDWARE}; @@ -186,8 +186,12 @@ pub fn run() -> sc_cli::Result<()> { .map_err(|lang_err| sc_cli::Error::Application(Box::new(lang_err))), None => { let runner = cli.create_runner(&cli.run)?; - runner - .run_node_until_exit(|config| async move { service::new_full(config).map_err(sc_cli::Error::Service) }) + runner.run_node_until_exit(|config| async move { + match cli.manual_seal { + true => test_service::new_test(config).map_err(sc_cli::Error::Service), + false => service::new_full(config).map_err(sc_cli::Error::Service), + } + }) } } } diff --git a/node/src/main.rs b/node/src/main.rs index b5a331cc..f4ebd751 100644 --- a/node/src/main.rs +++ b/node/src/main.rs @@ -1,10 +1,10 @@ -mod chain_spec; -#[macro_use] -mod service; mod benchmarking; +mod chain_spec; mod cli; mod command; mod rpc; +mod service; +mod test_service; fn main() -> sc_cli::Result<()> { command::run() diff --git a/node/src/rpc.rs b/node/src/rpc.rs index d9d930c3..3bb65b08 100644 --- a/node/src/rpc.rs +++ b/node/src/rpc.rs @@ -1,7 +1,9 @@ use std::sync::Arc; +use futures::channel::mpsc::Sender; use jsonrpsee::RpcModule; use sc_consensus_babe::BabeWorkerHandle; +use sc_consensus_manual_seal::{rpc::ManualSeal, rpc::ManualSealApiServer, EngineCommand}; pub use sc_rpc_api::DenyUnsafe; use sc_transaction_pool_api::TransactionPool; use sp_api::ProvideRuntimeApi; @@ -10,7 +12,7 @@ use sp_blockchain::{Error as BlockChainError, HeaderBackend, HeaderMetadata}; use sp_consensus::SelectChain; use sp_consensus_babe::BabeApi; -use dscp_node_runtime::{opaque::Block, AccountId, Balance, Index}; +use dscp_node_runtime::{opaque::Block, AccountId, Balance, Hash, Index}; use sp_keystore::KeystorePtr; /// Extra dependencies for BABE. @@ -74,3 +76,44 @@ where Ok(module) } + +/// Full client dependencies. +pub struct TestDeps { + /// The client instance to use. + pub client: Arc, + /// Transaction pool instance. + pub pool: Arc

, + /// Whether to deny unsafe calls + pub deny_unsafe: DenyUnsafe, + /// A command stream to send authoring commands to manual seal consensus engine + pub command_sink: Sender>, +} + +// Instantiate all full RPC extensions. +pub fn create_test(deps: TestDeps) -> Result, Box> +where + C: ProvideRuntimeApi, + C: HeaderBackend + HeaderMetadata + 'static, + C: Send + Sync + 'static, + C::Api: substrate_frame_rpc_system::AccountNonceApi, + C::Api: pallet_transaction_payment_rpc::TransactionPaymentRuntimeApi, + C::Api: BlockBuilder, + P: TransactionPool + 'static, +{ + use pallet_transaction_payment_rpc::{TransactionPayment, TransactionPaymentApiServer}; + use substrate_frame_rpc_system::{System, SystemApiServer}; + + let mut module = RpcModule::new(()); + let TestDeps { + client, + pool, + deny_unsafe, + command_sink, + } = deps; + + module.merge(System::new(client.clone(), pool, deny_unsafe).into_rpc())?; + module.merge(TransactionPayment::new(client.clone()).into_rpc())?; + module.merge(ManualSeal::new(command_sink).into_rpc())?; + + Ok(module) +} diff --git a/node/src/test_service.rs b/node/src/test_service.rs new file mode 100644 index 00000000..916baa2a --- /dev/null +++ b/node/src/test_service.rs @@ -0,0 +1,240 @@ +//! Service and ServiceFactory implementation. Specialized wrapper over substrate service. +#![allow(clippy::needless_borrow)] +use dscp_node_runtime::{self, opaque::Block, RuntimeApi}; +use futures::channel::mpsc::Receiver; +use sc_consensus_manual_seal::{ + consensus::{babe::BabeConsensusDataProvider, timestamp::SlotTimestampProvider}, + EngineCommand, ManualSealParams, +}; +pub use sc_executor::NativeElseWasmExecutor; +use sc_service::{error::Error as ServiceError, Configuration, PartialComponents, TaskManager}; +use sc_telemetry::{Telemetry, TelemetryWorker}; +use sp_consensus_babe::AuthorityId; +use sp_core::H256; +use sp_keyring::sr25519::Keyring::Alice; +use std::sync::Arc; + +// Our native executor instance. +pub struct ExecutorDispatch; + +impl sc_executor::NativeExecutionDispatch for ExecutorDispatch { + /// Only enable the benchmarking host functions when we actually want to benchmark. + #[cfg(feature = "runtime-benchmarks")] + type ExtendHostFunctions = frame_benchmarking::benchmarking::HostFunctions; + /// Otherwise we only use the default Substrate host functions. + #[cfg(not(feature = "runtime-benchmarks"))] + type ExtendHostFunctions = (); + + fn dispatch(method: &str, data: &[u8]) -> Option> { + dscp_node_runtime::api::dispatch(method, data) + } + + fn native_version() -> sc_executor::NativeVersion { + dscp_node_runtime::native_version() + } +} + +pub type FullClient = sc_service::TFullClient>; +type FullBackend = sc_service::TFullBackend; +type FullSelectChain = sc_consensus::LongestChain; +type FullGrandpaBlockImport = sc_consensus_grandpa::GrandpaBlockImport; + +/// Returns most parts of a service. Not enough to run a full chain, +/// But enough to perform chain operations like purge-chain +fn new_partial( + config: &Configuration, +) -> Result< + sc_service::PartialComponents< + FullClient, + FullBackend, + FullSelectChain, + sc_consensus::DefaultImportQueue, + sc_transaction_pool::FullPool, + ( + impl Fn( + crate::rpc::DenyUnsafe, + sc_rpc::SubscriptionTaskExecutor, + ) -> Result, sc_service::Error>, + sc_consensus_babe::BabeBlockImport, + sc_consensus_babe::BabeLink, + Receiver>, + Option, + ), + >, + ServiceError, +> { + let telemetry = config + .telemetry_endpoints + .clone() + .filter(|x| !x.is_empty()) + .map(|endpoints| -> Result<_, sc_telemetry::Error> { + let worker = TelemetryWorker::new(16)?; + let telemetry = worker.handle().new_telemetry(endpoints); + Ok((worker, telemetry)) + }) + .transpose()?; + + let executor = sc_service::new_native_or_wasm_executor(config); + + let (client, backend, keystore_container, task_manager) = sc_service::new_full_parts::( + config, + telemetry.as_ref().map(|(_, telemetry)| telemetry.handle()), + executor, + )?; + let client = Arc::new(client); + + let telemetry = telemetry.map(|(worker, telemetry)| { + task_manager.spawn_handle().spawn("telemetry", None, worker.run()); + telemetry + }); + + let select_chain = sc_consensus::LongestChain::new(backend.clone()); + + let transaction_pool = sc_transaction_pool::BasicPool::new_full( + config.transaction_pool.clone(), + config.role.is_authority().into(), + config.prometheus_registry(), + task_manager.spawn_essential_handle(), + client.clone(), + ); + + let (grandpa_block_import, ..) = sc_consensus_grandpa::block_import( + client.clone(), + &client.clone(), + select_chain.clone(), + telemetry.as_ref().map(|x| x.handle()), + )?; + + let (block_import, babe_link) = sc_consensus_babe::block_import( + sc_consensus_babe::configuration(&*client)?, + grandpa_block_import, + client.clone(), + )?; + + let import_queue = sc_consensus_manual_seal::import_queue( + Box::new(block_import.clone()), + &task_manager.spawn_essential_handle(), + config.prometheus_registry(), + ); + + let (command_sink, commands_stream) = futures::channel::mpsc::channel(1000); + let rpc_builder = { + let client = client.clone(); + let pool = transaction_pool.clone(); + Box::new(move |deny_unsafe, _| { + let deps = crate::rpc::TestDeps { + client: client.clone(), + pool: pool.clone(), + deny_unsafe, + command_sink: command_sink.clone(), + }; + + crate::rpc::create_test(deps).map_err(Into::into) + }) + }; + + Ok(PartialComponents { + client, + backend, + import_queue, + keystore_container, + task_manager, + transaction_pool, + select_chain, + other: (rpc_builder, block_import, babe_link, commands_stream, telemetry), + }) +} + +/// Builds a new service for a full client. +pub fn new_test(config: Configuration) -> Result { + let sc_service::PartialComponents { + client, + backend, + mut task_manager, + import_queue, + keystore_container, + select_chain, + transaction_pool, + other: (rpc_builder, block_import, babe_link, commands_stream, mut telemetry), + } = new_partial(&config)?; + + let (network, system_rpc_tx, tx_handler_controller, network_starter, sync_service) = + sc_service::build_network(sc_service::BuildNetworkParams { + config: &config, + client: client.clone(), + transaction_pool: transaction_pool.clone(), + spawn_handle: task_manager.spawn_handle(), + import_queue, + block_announce_validator_builder: None, + warp_sync_params: None, + })?; + + if config.offchain_worker.enabled { + sc_service::build_offchain_workers(&config, task_manager.spawn_handle(), client.clone(), network.clone()); + } + + let role = config.role.clone(); + let prometheus_registry = config.prometheus_registry().cloned(); + + let _rpc_handlers = sc_service::spawn_tasks(sc_service::SpawnTasksParams { + config, + backend, + client: client.clone(), + keystore: keystore_container.keystore(), + network: network.clone(), + rpc_builder: Box::new(rpc_builder), + transaction_pool: transaction_pool.clone(), + task_manager: &mut task_manager, + system_rpc_tx, + tx_handler_controller, + sync_service: sync_service.clone(), + telemetry: telemetry.as_mut(), + })?; + + if role.is_authority() { + let proposer = sc_basic_authorship::ProposerFactory::new( + task_manager.spawn_handle(), + client.clone(), + transaction_pool.clone(), + prometheus_registry.as_ref(), + telemetry.as_ref().map(Telemetry::handle), + ); + + let babe_data_provider = BabeConsensusDataProvider::new( + client.clone(), + keystore_container.keystore(), + babe_link.epoch_changes().clone(), + vec![(AuthorityId::from(Alice.public()), 1000)], + ) + .expect(""); + + // Background authorship future. + let authorship_future = sc_consensus_manual_seal::run_manual_seal(ManualSealParams { + block_import, + env: proposer, + client: client.clone(), + pool: transaction_pool.clone(), + commands_stream, + select_chain, + consensus_data_provider: Some(Box::new(babe_data_provider)), + create_inherent_data_providers: move |_, ()| { + let client = client.clone(); + async move { + // Ok(sp_timestamp::InherentDataProvider::from_system_time()) + let timestamp = + SlotTimestampProvider::new_babe(client.clone()).map_err(|err| format!("{:?}", err))?; + let babe = sp_consensus_babe::inherents::InherentDataProvider::new(timestamp.slot()); + Ok((timestamp, babe)) + } + }, + }); + + // we spawn the future on a background thread managed by service. + task_manager + .spawn_essential_handle() + .spawn_blocking("manual-seal", Some("block-authoring"), authorship_future); + }; + + network_starter.start_network(); + Ok(task_manager) +} diff --git a/runtime/Cargo.toml b/runtime/Cargo.toml index 3d48a2bf..1c9f0a81 100644 --- a/runtime/Cargo.toml +++ b/runtime/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "dscp-node-runtime" -version = "9.1.2" +version = "9.3.0" authors = ["Digital Catapult "] edition = "2021" license = "Apache-2.0" diff --git a/runtime/src/lib.rs b/runtime/src/lib.rs index cd77d593..fd576acd 100644 --- a/runtime/src/lib.rs +++ b/runtime/src/lib.rs @@ -78,7 +78,7 @@ pub const VERSION: RuntimeVersion = RuntimeVersion { spec_name: create_runtime_str!("dscp"), impl_name: create_runtime_str!("dscp"), authoring_version: 1, - spec_version: 912, + spec_version: 930, impl_version: 1, apis: RUNTIME_API_VERSIONS, transaction_version: 1, From 69627f24a3e03ffd67ea41837f8bcbb7b9138c3f Mon Sep 17 00:00:00 2001 From: Matthew Dean Date: Mon, 18 Dec 2023 13:59:24 +0000 Subject: [PATCH 2/2] Apt fix? --- .github/workflows/release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index b915593f..f3fd8fbd 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -71,7 +71,7 @@ jobs: rustup target add ${{ matrix.architectures.target-tupl }} --toolchain ${{ needs.get-version.outputs.toolchain }} rustup target add wasm32-unknown-unknown --toolchain ${{ needs.get-version.outputs.toolchain }} - name: install deps - run: sudo apt-get install ${{ matrix.architectures.dependencies }} + run: sudo apt-get update && sudo apt-get install ${{ matrix.architectures.dependencies }} - name: Install sccache env: TEMP: ${{ runner.temp }}